daffainfo
/
nuclei
mirror of https://github.com/daffainfo/nuclei.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1064 from projectdiscovery/colorize_responses 

Implementation for: Add coloring to debug information #999
dev
forgedhallpass 2021-10-13 20:51:19 +03:00 committed by GitHub
parent c0ef419048 d88ed67d2c
commit 46d0058470
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
42 changed files with 1077 additions and 948 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
v2/cmd/integration-test/http.go
View File

@ -7,7 +7,6 @@ import (
"net"
"net/http"
"net/http/httptest"
"net/http/httputil"
"strings"
"github.com/julienschmidt/httprouter"
@ -34,21 +33,12 @@ var httpTestcases = map[string]testutils.TestCase{
"http/request-condition-new.yaml": &httpRequestCondition{},
}
func httpDebugRequestDump(r *http.Request) {
if debug {
if dump, err := httputil.DumpRequest(r, true); err == nil {
fmt.Printf("\nRequest dump: \n%s\n\n", string(dump))
}
}
}
type httpGetHeaders struct{}
// Execute executes a test case and returns an error if occurred
func (h *httpGetHeaders) Execute(filePath string) error {
router := httprouter.New()
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
httpDebugRequestDump(r)
if strings.EqualFold(r.Header.Get("test"), "nuclei") {
fmt.Fprintf(w, "This is test headers matcher text")
}
@ -72,7 +62,6 @@ type httpGetQueryString struct{}
func (h *httpGetQueryString) Execute(filePath string) error {
router := httprouter.New()
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
httpDebugRequestDump(r)
if strings.EqualFold(r.URL.Query().Get("test"), "nuclei") {
fmt.Fprintf(w, "This is test querystring matcher text")
}
@ -96,11 +85,9 @@ type httpGetRedirects struct{}
func (h *httpGetRedirects) Execute(filePath string) error {
router := httprouter.New()
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
httpDebugRequestDump(r)
http.Redirect(w, r, "/redirected", http.StatusFound)
})
router.GET("/redirected", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
httpDebugRequestDump(r)
fmt.Fprintf(w, "This is test redirects matcher text")
})
ts := httptest.NewServer(router)
@ -122,7 +109,6 @@ type httpGet struct{}
func (h *httpGet) Execute(filePath string) error {
router := httprouter.New()
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
httpDebugRequestDump(r)
fmt.Fprintf(w, "This is test matcher text")
})
ts := httptest.NewServer(router)
@ -146,7 +132,6 @@ func (h *httpPostBody) Execute(filePath string) error {
var routerErr error
router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
httpDebugRequestDump(r)
if err := r.ParseForm(); err != nil {
routerErr = err
return
@ -179,8 +164,6 @@ func (h *httpPostJSONBody) Execute(filePath string) error {
var routerErr error
router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
httpDebugRequestDump(r)
type doc struct {
Username string `json:"username"`
Password string `json:"password"`
@ -218,7 +201,6 @@ func (h *httpPostMultipartBody) Execute(filePath string) error {
var routerErr error
router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
httpDebugRequestDump(r)
if err := r.ParseMultipartForm(1 * 1024); err != nil {
routerErr = err
return
@ -261,7 +243,6 @@ func (h *httpRawDynamicExtractor) Execute(filePath string) error {
var routerErr error
router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
httpDebugRequestDump(r)
if err := r.ParseForm(); err != nil {
routerErr = err
return
@ -271,7 +252,6 @@ func (h *httpRawDynamicExtractor) Execute(filePath string) error {
}
})
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
httpDebugRequestDump(r)
if strings.EqualFold(r.URL.Query().Get("username"), "nuclei") {
fmt.Fprintf(w, "Test is test-dynamic-extractor-raw matcher text")
}
@ -300,7 +280,6 @@ func (h *httpRawGetQuery) Execute(filePath string) error {
var routerErr error
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
httpDebugRequestDump(r)
if strings.EqualFold(r.URL.Query().Get("test"), "nuclei") {
fmt.Fprintf(w, "Test is test raw-get-query-matcher text")
}
@ -329,8 +308,6 @@ func (h *httpRawGet) Execute(filePath string) error {
var routerErr error
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
httpDebugRequestDump(r)
fmt.Fprintf(w, "Test is test raw-get-matcher text")
})
ts := httptest.NewServer(router)
@ -357,7 +334,6 @@ func (h *httpRawPayload) Execute(filePath string) error {
var routerErr error
router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
httpDebugRequestDump(r)
if err := r.ParseForm(); err != nil {
routerErr = err
return
@ -393,7 +369,6 @@ func (h *httpRawPostBody) Execute(filePath string) error {
var routerErr error
router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
httpDebugRequestDump(r)
if err := r.ParseForm(); err != nil {
routerErr = err
return
@ -426,7 +401,6 @@ func (h *httpRawCookieReuse) Execute(filePath string) error {
var routerErr error
router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
httpDebugRequestDump(r)
if err := r.ParseForm(); err != nil {
routerErr = err
return
@ -436,7 +410,6 @@ func (h *httpRawCookieReuse) Execute(filePath string) error {
}
})
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
httpDebugRequestDump(r)
if err := r.ParseForm(); err != nil {
routerErr = err
return
@ -500,11 +473,9 @@ func (h *httpRequestCondition) Execute(filePath string) error {
var routerErr error
router.GET("/200", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
httpDebugRequestDump(r)
w.WriteHeader(200)
})
router.GET("/400", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
httpDebugRequestDump(r)
w.WriteHeader(400)
})
ts := httptest.NewServer(router)

9
v2/cmd/integration-test/network.go
View File

@ -82,7 +82,14 @@ func (h *networkMultiStep) Execute(filePath string) error {
if routerErr != nil {
return routerErr
}
if len(results) != 1 {
var expectedResultsSize int
if debug {
expectedResultsSize = 3
} else {
expectedResultsSize = 1
}
if len(results) != expectedResultsSize {
return errIncorrectResultsCount(results)
}
return nil

4
v2/cmd/integration-test/workflow.go
View File

@ -23,7 +23,6 @@ type workflowBasic struct{}
func (h *workflowBasic) Execute(filePath string) error {
router := httprouter.New()
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
httpDebugRequestDump(r)
fmt.Fprintf(w, "This is test matcher text")
})
ts := httptest.NewServer(router)
@ -45,7 +44,6 @@ type workflowConditionMatched struct{}
func (h *workflowConditionMatched) Execute(filePath string) error {
router := httprouter.New()
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
httpDebugRequestDump(r)
fmt.Fprintf(w, "This is test matcher text")
})
ts := httptest.NewServer(router)
@ -67,7 +65,6 @@ type workflowConditionUnmatch struct{}
func (h *workflowConditionUnmatch) Execute(filePath string) error {
router := httprouter.New()
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
httpDebugRequestDump(r)
fmt.Fprintf(w, "This is test matcher text")
})
ts := httptest.NewServer(router)
@ -89,7 +86,6 @@ type workflowMatcherName struct{}
func (h *workflowMatcherName) Execute(filePath string) error {
router := httprouter.New()
router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
httpDebugRequestDump(r)
fmt.Fprintf(w, "This is test matcher text")
})
ts := httptest.NewServer(router)

80
v2/internal/runner/update.go
View File

@ -48,7 +48,7 @@ var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`)
//
// If the path exists but does not contain the latest version of public templates,
// the new version is downloaded from GitHub to the templates' directory, overwriting the old content.
func (r *Runner) updateTemplates() error {
func (r *Runner) updateTemplates() error { // TODO this method does more than just update templates. Should be refactored.
home, err := os.UserHomeDir()
if err != nil {
return err
@ -75,6 +75,7 @@ func (r *Runner) updateTemplates() error {
if r.options.NoUpdateTemplates && !r.options.UpdateTemplates {
return nil
}
client.InitNucleiVersion(config.Version)
r.fetchLatestVersionsFromGithub(configDir) // also fetch the latest versions
@ -121,59 +122,74 @@ func (r *Runner) updateTemplates() error {
return nil
}
// Get the configuration currently on disk.
verText := r.templatesConfig.TemplateVersion
indices := reVersion.FindStringIndex(verText)
if indices == nil {
return fmt.Errorf("invalid release found with tag %s", err)
}
if indices[0] > 0 {
verText = verText[indices[0]:]
}
oldVersion, err := semver.Make(verText)
latestVersion, currentVersion, err := getVersions(r)
if err != nil {
return err
}
version, err := semver.Parse(r.templatesConfig.NucleiTemplatesLatestVersion)
if err != nil {
return err
}
if version.EQ(oldVersion) {
if latestVersion.EQ(currentVersion) {
if r.options.UpdateTemplates {
gologger.Info().Msgf("No new updates found for nuclei templates")
}
return config.WriteConfiguration(r.templatesConfig)
}
if version.GT(oldVersion) {
gologger.Info().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String())
if err := updateTemplates(latestVersion, currentVersion, r, ctx); err != nil {
return err
}
return nil
}
func updateTemplates(latestVersion semver.Version, currentVersion semver.Version, runner *Runner, ctx context.Context) error {
if latestVersion.GT(currentVersion) {
gologger.Info().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", currentVersion, latestVersion.String())
gologger.Info().Msgf("Downloading latest release...")
if r.options.TemplatesDirectory != "" {
r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory
if runner.options.TemplatesDirectory != "" {
runner.templatesConfig.TemplatesDirectory = runner.options.TemplatesDirectory
}
r.templatesConfig.TemplateVersion = version.String()
runner.templatesConfig.TemplateVersion = latestVersion.String()
gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory)
gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", latestVersion.String(), runner.templatesConfig.TemplatesDirectory)
asset, err := r.getLatestReleaseFromGithub(r.templatesConfig.NucleiTemplatesLatestVersion)
asset, err := runner.getLatestReleaseFromGithub(runner.templatesConfig.NucleiTemplatesLatestVersion)
if err != nil {
return err
}
if _, err := r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()); err != nil {
if _, err := runner.downloadReleaseAndUnzip(ctx, latestVersion.String(), asset.GetZipballURL()); err != nil {
return err
}
if err := config.WriteConfiguration(r.templatesConfig); err != nil {
if err := config.WriteConfiguration(runner.templatesConfig); err != nil {
return err
}
gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). GoodLuck!\n", version.String())
gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). GoodLuck!\n", latestVersion.String())
}
return nil
}
func getVersions(runner *Runner) (semver.Version, semver.Version, error) {
// Get the configuration currently on disk.
verText := runner.templatesConfig.TemplateVersion
indices := reVersion.FindStringIndex(verText)
if indices == nil {
return semver.Version{}, semver.Version{}, fmt.Errorf("invalid release found with tag %s", verText)
}
if indices[0] > 0 {
verText = verText[indices[0]:]
}
currentVersion, err := semver.Make(verText)
if err != nil {
return semver.Version{}, semver.Version{}, err
}
latestVersion, err := semver.Parse(runner.templatesConfig.NucleiTemplatesLatestVersion)
if err != nil {
return semver.Version{}, semver.Version{}, err
}
return latestVersion, currentVersion, nil
}
// readInternalConfigurationFile reads the internal configuration file for nuclei
func (r *Runner) readInternalConfigurationFile(home, configDir string) error {
templatesConfigFile := filepath.Join(configDir, nucleiConfigFilename)
@ -210,9 +226,9 @@ func (r *Runner) checkNucleiIgnoreFileUpdates(configDir string) bool {
// getLatestReleaseFromGithub returns the latest release from GitHub
func (r *Runner) getLatestReleaseFromGithub(latestTag string) (*github.RepositoryRelease, error) {
client := github.NewClient(nil)
gitHubClient := github.NewClient(nil)
release, _, err := client.Repositories.GetReleaseByTag(context.Background(), userName, repoName, "v"+latestTag)
release, _, err := gitHubClient.Repositories.GetReleaseByTag(context.Background(), userName, repoName, "v"+latestTag)
if err != nil {
return nil, err
}
@ -361,7 +377,7 @@ func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdat
for templatePath, templateChecksums := range templateChecksumsMap {
_, ok := results.checksums[templatePath]
if !ok && templateChecksums[0] == templateChecksums[1] {
os.Remove(templatePath)
_ = os.Remove(templatePath)
results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(templatePath, r.templatesConfig.TemplatesDirectory), string(os.PathSeparator)))
}
}
@ -456,7 +472,7 @@ func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version st
// fetchLatestVersionsFromGithub fetches the latest versions of nuclei repos from GitHub
//
// This fetches latest nuclei/templates/ignore from https://version-check.nuclei.sh/versions
// This fetches the latest nuclei/templates/ignore from https://version-check.nuclei.sh/versions
// If you want to disable this automatic update check, use -nut flag.
func (r *Runner) fetchLatestVersionsFromGithub(configDir string) {
versions, err := client.GetLatestNucleiTemplatesVersion()

3
v2/pkg/catalog/config/config.go
View File

@ -6,8 +6,9 @@ import (
jsoniter "github.com/json-iterator/go"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"gopkg.in/yaml.v2"
"github.com/projectdiscovery/gologger"
)
// Config contains the internal nuclei engine configuration

50
v2/pkg/operators/matchers/match.go
View File

@ -4,6 +4,7 @@ import (
"encoding/hex"
"strings"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
)
@ -40,7 +41,8 @@ func (m *Matcher) MatchSize(length int) bool {
}
// MatchWords matches a word check against a corpus.
func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}) bool {
func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}) (bool, []string) {
var matchedWords []string
// Iterate over all the words accepted as valid
for i, word := range m.Words {
if dynamicValues == nil {
@ -57,7 +59,7 @@ func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}
// If we are in an AND request and a match failed,
// return false as the AND condition fails on any single mismatch.
if m.condition == ANDCondition {
return false
return false, []string{}
}
// Continue with the flow since it's an OR Condition.
continue
@ -65,19 +67,22 @@ func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}
// If the condition was an OR, return on the first match.
if m.condition == ORCondition {
return true
return true, []string{word}
}
matchedWords = append(matchedWords, word)
// If we are at the end of the words, return with true
if len(m.Words)-1 == i {
return true
return true, matchedWords
}
}
return false
return false, []string{}
}
// MatchRegex matches a regex check against a corpus
func (m *Matcher) MatchRegex(corpus string) bool {
func (m *Matcher) MatchRegex(corpus string) (bool, []string) {
var matchedRegexes []string
// Iterate over all the regexes accepted as valid
for i, regex := range m.regexCompiled {
// Continue if the regex doesn't match
@ -85,36 +90,47 @@ func (m *Matcher) MatchRegex(corpus string) bool {
// If we are in an AND request and a match failed,
// return false as the AND condition fails on any single mismatch.
if m.condition == ANDCondition {
return false
return false, []string{}
}
// Continue with the flow since it's an OR Condition.
continue
}
currentMatches := regex.FindAllString(corpus, -1)
// If the condition was an OR, return on the first match.
if m.condition == ORCondition {
return true
return true, currentMatches
}
matchedRegexes = append(matchedRegexes, currentMatches...)
// If we are at the end of the regex, return with true
if len(m.regexCompiled)-1 == i {
return true
return true, matchedRegexes
}
}
return false
return false, []string{}
}
// MatchBinary matches a binary check against a corpus
func (m *Matcher) MatchBinary(corpus string) bool {
func (m *Matcher) MatchBinary(corpus string) (bool, []string) {
var matchedBinary []string
// Iterate over all the words accepted as valid
for i, binary := range m.Binary {
// Continue if the word doesn't match
hexa, _ := hex.DecodeString(binary)
hexa, err := hex.DecodeString(binary)
if err != nil {
gologger.Warning().Msgf("Could not hex encode the given binary matcher value: '%s'", binary)
if m.condition == ANDCondition {
return false, []string{}
}
continue
}
if !strings.Contains(corpus, string(hexa)) {
// If we are in an AND request and a match failed,
// return false as the AND condition fails on any single mismatch.
if m.condition == ANDCondition {
return false
return false, []string{}
}
// Continue with the flow since it's an OR Condition.
continue
@ -122,15 +138,17 @@ func (m *Matcher) MatchBinary(corpus string) bool {
// If the condition was an OR, return on the first match.
if m.condition == ORCondition {
return true
return true, []string{string(hexa)}
}
matchedBinary = append(matchedBinary, string(hexa))
// If we are at the end of the words, return with true
if len(m.Binary)-1 == i {
return true
return true, matchedBinary
}
}
return false
return false, []string{}
}
// MatchDSL matches on a generic map result

60
v2/pkg/operators/matchers/match_test.go
View File

@ -6,27 +6,60 @@ import (
"github.com/stretchr/testify/require"
)
func TestANDCondition(t *testing.T) {
func TestWordANDCondition(t *testing.T) {
m := &Matcher{condition: ANDCondition, Words: []string{"a", "b"}}
matched := m.MatchWords("a b", nil)
require.True(t, matched, "Could not match valid AND condition")
isMatched, matched := m.MatchWords("a b", nil)
require.True(t, isMatched, "Could not match words with valid AND condition")
require.Equal(t, m.Words, matched)
matched = m.MatchWords("b", nil)
require.False(t, matched, "Could match invalid AND condition")
isMatched, matched = m.MatchWords("b", nil)
require.False(t, isMatched, "Could match words with invalid AND condition")
require.Equal(t, []string{}, matched)
}
func TestRegexANDCondition(t *testing.T) {
m := &Matcher{Type: "regex", Condition: "and", Regex: []string{"[a-z]{3}", "\\d{2}"}}
err := m.CompileMatchers()
require.Nil(t, err)
isMatched, matched := m.MatchRegex("abc abcd 123")
require.True(t, isMatched, "Could not match regex with valid AND condition")
require.Equal(t, []string{"abc", "abc", "12"}, matched)
isMatched, matched = m.MatchRegex("bc 1")
require.False(t, isMatched, "Could match regex with invalid AND condition")
require.Equal(t, []string{}, matched)
}
func TestORCondition(t *testing.T) {
m := &Matcher{condition: ORCondition, Words: []string{"a", "b"}}
matched := m.MatchWords("a b", nil)
require.True(t, matched, "Could not match valid OR condition")
isMatched, matched := m.MatchWords("a b", nil)
require.True(t, isMatched, "Could not match valid word OR condition")
require.Equal(t, []string{"a"}, matched)
matched = m.MatchWords("b", nil)
require.True(t, matched, "Could not match valid OR condition")
isMatched, matched = m.MatchWords("b", nil)
require.True(t, isMatched, "Could not match valid word OR condition")
require.Equal(t, []string{"b"}, matched)
matched = m.MatchWords("c", nil)
require.False(t, matched, "Could match invalid OR condition")
isMatched, matched = m.MatchWords("c", nil)
require.False(t, isMatched, "Could match invalid word OR condition")
require.Equal(t, []string{}, matched)
}
func TestRegexOrCondition(t *testing.T) {
m := &Matcher{Type: "regex", Condition: "or", Regex: []string{"[a-z]{3}", "\\d{2}"}}
err := m.CompileMatchers()
require.Nil(t, err)
isMatched, matched := m.MatchRegex("ab 123")
require.True(t, isMatched, "Could not match valid regex OR condition")
require.Equal(t, []string{"12"}, matched)
isMatched, matched = m.MatchRegex("bc 1")
require.False(t, isMatched, "Could match invalid regex OR condition")
require.Equal(t, []string{}, matched)
}
func TestHexEncoding(t *testing.T) {
@ -34,6 +67,7 @@ func TestHexEncoding(t *testing.T) {
err := m.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := m.MatchWords("PING", nil)
require.True(t, matched, "Could not match valid Hex condition")
isMatched, matched := m.MatchWords("PING", nil)
require.True(t, isMatched, "Could not match valid Hex condition")
require.Equal(t, m.Words, matched)
}

8
v2/pkg/operators/matchers/matchers.go
View File

@ -165,6 +165,14 @@ func (m *Matcher) Result(data bool) bool {
return data
}
// ResultWithMatchedSnippet returns true and the matched snippet, or false and an empty string
func (m *Matcher) ResultWithMatchedSnippet(data bool, matchedSnippet []string) (bool, []string) {
if m.Negative {
return !data, []string{}
}
return data, matchedSnippet
}
// GetType returns the type of the matcher
func (m *Matcher) GetType() MatcherType {
return m.matcherType

71
v2/pkg/operators/operators.go
View File

@ -1,6 +1,8 @@
package operators
import (
"strconv"
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
@ -32,19 +34,19 @@ type Operators struct {
}
// Compile compiles the operators as well as their corresponding matchers and extractors
func (r *Operators) Compile() error {
if r.MatchersCondition != "" {
r.matchersCondition = matchers.ConditionTypes[r.MatchersCondition]
func (operators *Operators) Compile() error {
if operators.MatchersCondition != "" {
operators.matchersCondition = matchers.ConditionTypes[operators.MatchersCondition]
} else {
r.matchersCondition = matchers.ORCondition
operators.matchersCondition = matchers.ORCondition
}
for _, matcher := range r.Matchers {
for _, matcher := range operators.Matchers {
if err := matcher.CompileMatchers(); err != nil {
return errors.Wrap(err, "could not compile matcher")
}
}
for _, extractor := range r.Extractors {
for _, extractor := range operators.Extractors {
if err := extractor.CompileExtractors(); err != nil {
return errors.Wrap(err, "could not compile extractor")
}
@ -53,8 +55,8 @@ func (r *Operators) Compile() error {
}
// GetMatchersCondition returns the condition for the matchers
func (r *Operators) GetMatchersCondition() matchers.ConditionType {
return r.matchersCondition
func (operators *Operators) GetMatchersCondition() matchers.ConditionType {
return operators.matchersCondition
}
// Result is a result structure created from operators running on data.
@ -64,7 +66,7 @@ type Result struct {
// Extracted is true if any result type values were extracted
Extracted bool
// Matches is a map of matcher names that we matched
Matches map[string]struct{}
Matches map[string][]string
// Extracts contains all the data extracted from inputs
Extracts map[string][]string
// OutputExtracts is the list of extracts to be displayed on screen.
@ -100,24 +102,24 @@ func (r *Result) Merge(result *Result) {
}
// MatchFunc performs matching operation for a matcher on model and returns true or false.
type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) bool
type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string)
// ExtractFunc performs extracting operation for an extractor on model and returns true or false.
type ExtractFunc func(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{}
// Execute executes the operators on data and returns a result structure
func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extract ExtractFunc) (*Result, bool) {
matcherCondition := r.GetMatchersCondition()
func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc, extract ExtractFunc, isDebug bool) (*Result, bool) {
matcherCondition := operators.GetMatchersCondition()
var matches bool
result := &Result{
Matches: make(map[string]struct{}),
Matches: make(map[string][]string),
Extracts: make(map[string][]string),
DynamicValues: make(map[string]interface{}),
}
// Start with the extractors first and evaluate them.
for _, extractor := range r.Extractors {
for _, extractor := range operators.Extractors {
var extractorResults []string
for match := range extract(data, extractor) {
@ -136,24 +138,23 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac
}
}
for _, matcher := range r.Matchers {
// Check if the matcher matched
if !match(data, matcher) {
// If the condition is AND we haven't matched, try next request.
if matcherCondition == matchers.ANDCondition {
for matcherIndex, matcher := range operators.Matchers {
if isMatch, matched := match(data, matcher); isMatch {
if isDebug { // matchers without an explicit name or with AND condition should only be made visible if debug is enabled
matcherName := getMatcherName(matcher, matcherIndex)
result.Matches[matcherName] = matched
} else { // if it's a "named" matcher with OR condition, then display it
if matcherCondition == matchers.ORCondition && matcher.Name != "" {
result.Matches[matcher.Name] = matched
}
}
matches = true
} else if matcherCondition == matchers.ANDCondition {
if len(result.DynamicValues) > 0 {
return result, true
}
return nil, false
}
} else {
// If the matcher has matched, and it's an OR
// write the first output then move to next matcher.
if matcherCondition == matchers.ORCondition && matcher.Name != "" {
result.Matches[matcher.Name] = struct{}{}
}
matches = true
}
}
result.Matched = matches
@ -162,8 +163,8 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac
return result, true
}
// Don't print if we have matchers and they have not matched, regardless of extractor
if len(r.Matchers) > 0 && !matches {
// Don't print if we have matchers, and they have not matched, regardless of extractor
if len(operators.Matchers) > 0 && !matches {
return nil, false
}
// Write a final string of output if matcher type is
@ -174,12 +175,20 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac
return nil, false
}
func getMatcherName(matcher *matchers.Matcher, matcherIndex int) string {
if matcher.Name != "" {
return matcher.Name
} else {
return matcher.Type + "-" + strconv.Itoa(matcherIndex+1) // making the index start from 1 to be more readable
}
}
// ExecuteInternalExtractors executes internal dynamic extractors
func (r *Operators) ExecuteInternalExtractors(data map[string]interface{}, extract ExtractFunc) map[string]interface{} {
func (operators *Operators) ExecuteInternalExtractors(data map[string]interface{}, extract ExtractFunc) map[string]interface{} {
dynamicValues := make(map[string]interface{})
// Start with the extractors first and evaluate them.
for _, extractor := range r.Extractors {
for _, extractor := range operators.Extractors {
if !extractor.Internal {
continue
}

4
v2/pkg/protocols/common/clusterer/executer.go
View File

@ -67,7 +67,7 @@ func (e *Executer) Execute(input string) (bool, error) {
dynamicValues := make(map[string]interface{})
err := e.requests.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) {
for _, operator := range e.operators {
result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract)
result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse)
if matched && result != nil {
event.OperatorsResult = result
event.InternalEvent["template-id"] = operator.templateID
@ -98,7 +98,7 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve
dynamicValues := make(map[string]interface{})
err := e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) {
for _, operator := range e.operators {
result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract)
result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse)
if matched && result != nil {
event.OperatorsResult = result
event.InternalEvent["template-id"] = operator.templateID

29
v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go Normal file
View File

@ -0,0 +1,29 @@
package eventcreator
import (
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
)
// CreateEvent wraps the outputEvent with the result of the operators defined on the request
func CreateEvent(request protocols.Request, outputEvent output.InternalEvent, isResponseDebug bool) *output.InternalWrappedEvent {
return CreateEventWithAdditionalOptions(request, outputEvent, isResponseDebug, func(internalWrappedEvent *output.InternalWrappedEvent) {})
}
// CreateEventWithAdditionalOptions wraps the outputEvent with the result of the operators defined on the request and enables extending the resulting event with additional attributes or values.
func CreateEventWithAdditionalOptions(request protocols.Request, outputEvent output.InternalEvent, isResponseDebug bool,
addAdditionalOptions func(internalWrappedEvent *output.InternalWrappedEvent)) *output.InternalWrappedEvent {
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
for _, compiledOperator := range request.GetCompiledOperators() {
if compiledOperator != nil {
result, ok := compiledOperator.Execute(outputEvent, request.Match, request.Extract, isResponseDebug)
if ok && result != nil {
event.OperatorsResult = result
addAdditionalOptions(event)
event.Results = append(event.Results, request.MakeResultEvent(event)...)
}
}
}
return event
}

35
v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go Normal file
View File

@ -0,0 +1,35 @@
package responsehighlighter
import (
"strconv"
"strings"
"github.com/logrusorgru/aurora"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
)
var colorizer = aurora.NewAurora(true)
func Highlight(operatorResult *operators.Result, response string, noColor bool) string {
result := response
if operatorResult != nil && !noColor {
for _, matches := range operatorResult.Matches {
if len(matches) > 0 {
for _, currentMatch := range matches {
result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String())
}
}
}
}
return result
}
func CreateStatusCodeSnippet(response string, statusCode int) string {
if strings.HasPrefix(response, "HTTP/") {
strStatusCode := strconv.Itoa(statusCode)
return response[:strings.Index(response, strStatusCode)+len(strStatusCode)]
}
return ""
}

2
v2/pkg/protocols/common/interactsh/interactsh.go
View File

@ -145,7 +145,7 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d
data.Event.InternalEvent["interactsh_protocol"] = interaction.Protocol
data.Event.InternalEvent["interactsh_request"] = interaction.RawRequest
data.Event.InternalEvent["interactsh_response"] = interaction.RawResponse
result, matched := data.Operators.Execute(data.Event.InternalEvent, data.MatchFunc, data.ExtractFunc)
result, matched := data.Operators.Execute(data.Event.InternalEvent, data.MatchFunc, data.ExtractFunc, false)
if !matched || result == nil {
return false // if we don't match, return
}

46
v2/pkg/protocols/dns/dns.go
View File

@ -76,47 +76,51 @@ type Request struct {
Resolvers []string `yaml:"resolvers,omitempty" jsonschema:"title=Resolvers,description=Define resolvers to use within the template"`
}
func (request *Request) GetCompiledOperators() []*operators.Operators {
return []*operators.Operators{request.CompiledOperators}
}
// GetID returns the unique ID of the request if any.
func (r *Request) GetID() string {
return r.ID
func (request *Request) GetID() string {
return request.ID
}
// Compile compiles the protocol request for further execution.
func (r *Request) Compile(options *protocols.ExecuterOptions) error {
func (request *Request) Compile(options *protocols.ExecuterOptions) error {
dnsClientOptions := &dnsclientpool.Configuration{
Retries: r.Retries,
Retries: request.Retries,
}
if len(r.Resolvers) > 0 {
dnsClientOptions.Resolvers = r.Resolvers
if len(request.Resolvers) > 0 {
dnsClientOptions.Resolvers = request.Resolvers
}
// Create a dns client for the class
client, err := dnsclientpool.Get(options.Options, dnsClientOptions)
if err != nil {
return errors.Wrap(err, "could not get dns client")
}
r.dnsClient = client
request.dnsClient = client
if len(r.Matchers) > 0 || len(r.Extractors) > 0 {
compiled := &r.Operators
if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
compiled := &request.Operators
if err := compiled.Compile(); err != nil {
return errors.Wrap(err, "could not compile operators")
}
r.CompiledOperators = compiled
request.CompiledOperators = compiled
}
r.class = classToInt(r.Class)
r.options = options
r.question = questionTypeToInt(r.Type)
request.class = classToInt(request.Class)
request.options = options
request.question = questionTypeToInt(request.Type)
return nil
}
// Requests returns the total number of requests the YAML rule will perform
func (r *Request) Requests() int {
func (request *Request) Requests() int {
return 1
}
// Make returns the request to be sent for the protocol
func (r *Request) Make(domain string) (*dns.Msg, error) {
if r.question != dns.TypePTR && net.ParseIP(domain) != nil {
func (request *Request) Make(domain string) (*dns.Msg, error) {
if request.question != dns.TypePTR && net.ParseIP(domain) != nil {
return nil, errors.New("cannot use IP address as DNS input")
}
domain = dns.Fqdn(domain)
@ -124,20 +128,20 @@ func (r *Request) Make(domain string) (*dns.Msg, error) {
// Build a request on the specified URL
req := new(dns.Msg)
req.Id = dns.Id()
req.RecursionDesired = r.Recursion
req.RecursionDesired = request.Recursion
var q dns.Question
final := replacer.Replace(r.Name, map[string]interface{}{"FQDN": domain})
final := replacer.Replace(request.Name, map[string]interface{}{"FQDN": domain})
q.Name = dns.Fqdn(final)
q.Qclass = r.class
q.Qtype = r.question
q.Qclass = request.class
q.Qtype = request.question
req.Question = append(req.Question, q)
req.SetEdns0(4096, false)
switch r.question {
switch request.question {
case dns.TypeTXT:
req.AuthenticatedData = true
}

133
v2/pkg/protocols/dns/operators.go
View File

@ -10,11 +10,12 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
// Match matches a generic data response again a given matcher
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool {
func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
partString := matcher.Part
switch partString {
case "body", "all", "":
@ -23,28 +24,32 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher)
item, ok := data[partString]
if !ok {
return false
return false, []string{}
}
switch matcher.GetType() {
case matchers.StatusMatcher:
return matcher.Result(matcher.MatchStatusCode(item.(int)))
case matchers.SizeMatcher:
return matcher.Result(matcher.MatchSize(len(types.ToString(item))))
case matchers.WordsMatcher:
return matcher.Result(matcher.MatchWords(types.ToString(item), nil))
case matchers.RegexMatcher:
return matcher.Result(matcher.MatchRegex(types.ToString(item)))
case matchers.BinaryMatcher:
return matcher.Result(matcher.MatchBinary(types.ToString(item)))
case matchers.DSLMatcher:
return matcher.Result(matcher.MatchDSL(data))
statusCode, ok := item.(int)
if !ok {
return false, []string{}
}
return false
return matcher.Result(matcher.MatchStatusCode(statusCode)), []string{}
case matchers.SizeMatcher:
return matcher.Result(matcher.MatchSize(len(types.ToString(item)))), []string{}
case matchers.WordsMatcher:
return matcher.ResultWithMatchedSnippet(matcher.MatchWords(types.ToString(item), nil))
case matchers.RegexMatcher:
return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(types.ToString(item)))
case matchers.BinaryMatcher:
return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(types.ToString(item)))
case matchers.DSLMatcher:
return matcher.Result(matcher.MatchDSL(data)), []string{}
}
return false, []string{}
}
// Extract performs extracting operation for an extractor on model and returns true or false.
func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
part := extractor.Part
switch part {
case "body", "all":
@ -67,77 +72,29 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext
}
// responseToDSLMap converts a DNS response to a map for use in DSL matching
func (r *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) output.InternalEvent {
data := make(output.InternalEvent, 11)
// Some data regarding the request metadata
data["host"] = host
data["matched"] = matched
data["request"] = req.String()
data["rcode"] = resp.Rcode
buffer := &bytes.Buffer{}
for _, question := range resp.Question {
buffer.WriteString(question.String())
func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) output.InternalEvent {
return output.InternalEvent{
"host": host,
"matched": matched,
"request": req.String(),
"rcode": resp.Rcode,
"question": questionToString(resp.Question),
"extra": rrToString(resp.Extra),
"answer": rrToString(resp.Answer),
"ns": rrToString(resp.Ns),
"raw": resp.String(),
"template-id": request.options.TemplateID,
"template-info": request.options.TemplateInfo,
"template-path": request.options.TemplatePath,
}
data["question"] = buffer.String()
buffer.Reset()
for _, extra := range resp.Extra {
buffer.WriteString(extra.String())
}
data["extra"] = buffer.String()
buffer.Reset()
for _, answer := range resp.Answer {
buffer.WriteString(answer.String())
}
data["answer"] = buffer.String()
buffer.Reset()
for _, ns := range resp.Ns {
buffer.WriteString(ns.String())
}
data["ns"] = buffer.String()
buffer.Reset()
rawData := resp.String()
data["raw"] = rawData
data["template-id"] = r.options.TemplateID
data["template-info"] = r.options.TemplateInfo
data["template-path"] = r.options.TemplatePath
return data
}
// MakeResultEvent creates a result event from internal wrapped event
func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
if len(wrapped.OperatorsResult.DynamicValues) > 0 {
return nil
}
results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1)
// If we have multiple matchers with names, write each of them separately.
if len(wrapped.OperatorsResult.Matches) > 0 {
for k := range wrapped.OperatorsResult.Matches {
data := r.makeResultEventItem(wrapped)
data.MatcherName = k
results = append(results, data)
}
} else if len(wrapped.OperatorsResult.Extracts) > 0 {
for k, v := range wrapped.OperatorsResult.Extracts {
data := r.makeResultEventItem(wrapped)
data.ExtractedResults = v
data.ExtractorName = k
results = append(results, data)
}
} else {
data := r.makeResultEventItem(wrapped)
results = append(results, data)
}
return results
func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
return protocols.MakeDefaultResultEvent(request, wrapped)
}
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
@ -152,3 +109,19 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out
}
return data
}
func rrToString(resourceRecords []dns.RR) string { // TODO rewrite with generics when available
buffer := &bytes.Buffer{}
for _, resourceRecord := range resourceRecords {
buffer.WriteString(resourceRecord.String())
}
return buffer.String()
}
func questionToString(resourceRecords []dns.Question) string {
buffer := &bytes.Buffer{}
for _, resourceRecord := range resourceRecords {
buffer.WriteString(resourceRecord.String())
}
return buffer.String()
}

28
v2/pkg/protocols/dns/operators_test.go
View File

@ -87,8 +87,9 @@ func TestDNSOperatorMatch(t *testing.T) {
err = matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid response")
isMatch, matched := request.Match(event, matcher)
require.True(t, isMatch, "could not match valid response")
require.Equal(t, matcher.Words, matched)
})
t.Run("rcode", func(t *testing.T) {
@ -100,8 +101,9 @@ func TestDNSOperatorMatch(t *testing.T) {
err = matcher.CompileMatchers()
require.Nil(t, err, "could not compile rcode matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid rcode response")
isMatched, matched := request.Match(event, matcher)
require.True(t, isMatched, "could not match valid rcode response")
require.Equal(t, []string{}, matched)
})
t.Run("negative", func(t *testing.T) {
@ -114,8 +116,9 @@ func TestDNSOperatorMatch(t *testing.T) {
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile negative matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid negative response matcher")
isMatched, matched := request.Match(event, matcher)
require.True(t, isMatched, "could not match valid negative response matcher")
require.Equal(t, []string{}, matched)
})
t.Run("invalid", func(t *testing.T) {
@ -127,8 +130,9 @@ func TestDNSOperatorMatch(t *testing.T) {
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.False(t, matched, "could match invalid response matcher")
isMatched, matched := request.Match(event, matcher)
require.False(t, isMatched, "could match invalid response matcher")
require.Equal(t, []string{}, matched)
})
}
@ -232,13 +236,15 @@ func TestDNSMakeResult(t *testing.T) {
event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one")
finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
if request.CompiledOperators != nil {
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract)
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, false)
if ok && result != nil {
finalEvent.OperatorsResult = result
finalEvent.Results = request.MakeResultEvent(finalEvent)
}
}
require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
require.Equal(t, "1.1.1.1", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
resultEvent := finalEvent.Results[0]
require.Equal(t, "test", resultEvent.MatcherName, "could not get correct matcher name of results")
require.Equal(t, "1.1.1.1", resultEvent.ExtractedResults[0], "could not get correct extracted results")
require.Equal(t, "one.one.one.one", resultEvent.Matched, "could not get matched value")
}

48
v2/pkg/protocols/dns/request.go
View File

@ -4,16 +4,19 @@ import (
"net/url"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
)
var _ protocols.Request = &Request{}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
// Parse the URL and return domain if URL.
var domain string
if isURL(input) {
@ -23,54 +26,49 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused
}
// Compile each request for the template based on the URL
compiledRequest, err := r.Make(domain)
compiledRequest, err := request.Make(domain)
if err != nil {
r.options.Output.Request(r.options.TemplateID, domain, "dns", err)
r.options.Progress.IncrementFailedRequestsBy(1)
request.options.Output.Request(request.options.TemplateID, domain, "dns", err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not build request")
}
requestString := compiledRequest.String()
if varErr := expressions.ContainsUnresolvedVariables(requestString); varErr != nil {
gologger.Warning().Msgf("[%s] Could not make dns request for %s: %v\n", r.options.TemplateID, domain, varErr)
gologger.Warning().Msgf("[%s] Could not make dns request for %s: %v\n", request.options.TemplateID, domain, varErr)
return nil
}
if r.options.Options.Debug || r.options.Options.DebugRequests {
gologger.Info().Str("domain", domain).Msgf("[%s] Dumped DNS request for %s", r.options.TemplateID, domain)
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Info().Str("domain", domain).Msgf("[%s] Dumped DNS request for %s", request.options.TemplateID, domain)
gologger.Print().Msgf("%s", requestString)
}
// Send the request to the target servers
resp, err := r.dnsClient.Do(compiledRequest)
resp, err := request.dnsClient.Do(compiledRequest)
if err != nil {
r.options.Output.Request(r.options.TemplateID, domain, "dns", err)
r.options.Progress.IncrementFailedRequestsBy(1)
request.options.Output.Request(request.options.TemplateID, domain, "dns", err)
request.options.Progress.IncrementFailedRequestsBy(1)
}
if resp == nil {
return errors.Wrap(err, "could not send dns request")
}
r.options.Progress.IncrementRequests()
request.options.Progress.IncrementRequests()
r.options.Output.Request(r.options.TemplateID, domain, "dns", err)
gologger.Verbose().Msgf("[%s] Sent DNS request to %s", r.options.TemplateID, domain)
request.options.Output.Request(request.options.TemplateID, domain, "dns", err)
gologger.Verbose().Msgf("[%s] Sent DNS request to %s", request.options.TemplateID, domain)
if r.options.Options.Debug || r.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped DNS response for %s", r.options.TemplateID, domain)
gologger.Print().Msgf("%s", resp.String())
}
outputEvent := r.responseToDSLMap(compiledRequest, resp, input, input)
outputEvent := request.responseToDSLMap(compiledRequest, resp, input, input)
for k, v := range previous {
outputEvent[k] = v
}
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if r.CompiledOperators != nil {
result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract)
if ok && result != nil {
event.OperatorsResult = result
event.Results = r.MakeResultEvent(event)
}
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped DNS response for %s", request.options.TemplateID, domain)
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, resp.String(), request.options.Options.NoColor))
}
callback(event)
return nil
}

36
v2/pkg/protocols/file/file.go
View File

@ -56,54 +56,54 @@ type Request struct {
var defaultDenylist = []string{".3g2", ".3gp", ".7z", ".apk", ".arj", ".avi", ".axd", ".bmp", ".css", ".csv", ".deb", ".dll", ".doc", ".drv", ".eot", ".exe", ".flv", ".gif", ".gifv", ".gz", ".h264", ".ico", ".iso", ".jar", ".jpeg", ".jpg", ".lock", ".m4a", ".m4v", ".map", ".mkv", ".mov", ".mp3", ".mp4", ".mpeg", ".mpg", ".msi", ".ogg", ".ogm", ".ogv", ".otf", ".pdf", ".pkg", ".png", ".ppt", ".psd", ".rar", ".rm", ".rpm", ".svg", ".swf", ".sys", ".tar.gz", ".tar", ".tif", ".tiff", ".ttf", ".vob", ".wav", ".webm", ".wmv", ".woff", ".woff2", ".xcf", ".xls", ".xlsx", ".zip"}
// GetID returns the unique ID of the request if any.
func (r *Request) GetID() string {
return r.ID
func (request *Request) GetID() string {
return request.ID
}
// Compile compiles the protocol request for further execution.
func (r *Request) Compile(options *protocols.ExecuterOptions) error {
if len(r.Matchers) > 0 || len(r.Extractors) > 0 {
compiled := &r.Operators
func (request *Request) Compile(options *protocols.ExecuterOptions) error {
if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
compiled := &request.Operators
if err := compiled.Compile(); err != nil {
return errors.Wrap(err, "could not compile operators")
}
r.CompiledOperators = compiled
request.CompiledOperators = compiled
}
// By default, use 5 MB as max size to read.
if r.MaxSize == 0 {
r.MaxSize = 5 * 1024 * 1024
if request.MaxSize == 0 {
request.MaxSize = 5 * 1024 * 1024
}
r.options = options
request.options = options
r.extensions = make(map[string]struct{})
r.extensionDenylist = make(map[string]struct{})
request.extensions = make(map[string]struct{})
request.extensionDenylist = make(map[string]struct{})
for _, extension := range r.Extensions {
for _, extension := range request.Extensions {
if extension == "all" {
r.allExtensions = true
request.allExtensions = true
} else {
if !strings.HasPrefix(extension, ".") {
extension = "." + extension
}
r.extensions[extension] = struct{}{}
request.extensions[extension] = struct{}{}
}
}
for _, extension := range defaultDenylist {
if !strings.HasPrefix(extension, ".") {
extension = "." + extension
}
r.extensionDenylist[extension] = struct{}{}
request.extensionDenylist[extension] = struct{}{}
}
for _, extension := range r.ExtensionDenylist {
for _, extension := range request.ExtensionDenylist {
if !strings.HasPrefix(extension, ".") {
extension = "." + extension
}
r.extensionDenylist[extension] = struct{}{}
request.extensionDenylist[extension] = struct{}{}
}
return nil
}
// Requests returns the total number of requests the YAML rule will perform
func (r *Request) Requests() int {
func (request *Request) Requests() int {
return 1
}

35
v2/pkg/protocols/file/find.go
View File

@ -7,50 +7,51 @@ import (
"github.com/karrick/godirwalk"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
)
// getInputPaths parses the specified input paths and returns a compiled
// list of finished absolute paths to the files evaluating any allowlist, denylist,
// glob, file or folders, etc.
func (r *Request) getInputPaths(target string, callback func(string)) error {
func (request *Request) getInputPaths(target string, callback func(string)) error {
processed := make(map[string]struct{})
// Template input includes a wildcard
if strings.Contains(target, "*") && !r.NoRecursive {
if err := r.findGlobPathMatches(target, processed, callback); err != nil {
if strings.Contains(target, "*") && !request.NoRecursive {
if err := request.findGlobPathMatches(target, processed, callback); err != nil {
return errors.Wrap(err, "could not find glob matches")
}
return nil
}
// Template input is either a file or a directory
file, err := r.findFileMatches(target, processed, callback)
file, err := request.findFileMatches(target, processed, callback)
if err != nil {
return errors.Wrap(err, "could not find file")
}
if file {
return nil
}
if r.NoRecursive {
if request.NoRecursive {
return nil // we don't process dirs in no-recursive mode
}
// Recursively walk down the Templates directory and run all
// the template file checks
if err := r.findDirectoryMatches(target, processed, callback); err != nil {
if err := request.findDirectoryMatches(target, processed, callback); err != nil {
return errors.Wrap(err, "could not find directory matches")
}
return nil
}
// findGlobPathMatches returns the matched files from a glob path
func (r *Request) findGlobPathMatches(absPath string, processed map[string]struct{}, callback func(string)) error {
func (request *Request) findGlobPathMatches(absPath string, processed map[string]struct{}, callback func(string)) error {
matches, err := filepath.Glob(absPath)
if err != nil {
return errors.Errorf("wildcard found, but unable to glob: %s\n", err)
}
for _, match := range matches {
if !r.validatePath(match) {
if !request.validatePath(match) {
continue
}
if _, ok := processed[match]; !ok {
@ -63,7 +64,7 @@ func (r *Request) findGlobPathMatches(absPath string, processed map[string]struc
// findFileMatches finds if a path is an absolute file. If the path
// is a file, it returns true otherwise false with no errors.
func (r *Request) findFileMatches(absPath string, processed map[string]struct{}, callback func(string)) (bool, error) {
func (request *Request) findFileMatches(absPath string, processed map[string]struct{}, callback func(string)) (bool, error) {
info, err := os.Stat(absPath)
if err != nil {
return false, err
@ -72,7 +73,7 @@ func (r *Request) findFileMatches(absPath string, processed map[string]struct{},
return false, nil
}
if _, ok := processed[absPath]; !ok {
if !r.validatePath(absPath) {
if !request.validatePath(absPath) {
return false, nil
}
processed[absPath] = struct{}{}
@ -82,7 +83,7 @@ func (r *Request) findFileMatches(absPath string, processed map[string]struct{},
}
// findDirectoryMatches finds matches for templates from a directory
func (r *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error {
func (request *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error {
err := godirwalk.Walk(absPath, &godirwalk.Options{
Unsorted: true,
ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {
@ -92,7 +93,7 @@ func (r *Request) findDirectoryMatches(absPath string, processed map[string]stru
if d.IsDir() {
return nil
}
if !r.validatePath(path) {
if !request.validatePath(path) {
return nil
}
if _, ok := processed[path]; !ok {
@ -106,17 +107,17 @@ func (r *Request) findDirectoryMatches(absPath string, processed map[string]stru
}
// validatePath validates a file path for blacklist and whitelist options
func (r *Request) validatePath(item string) bool {
func (request *Request) validatePath(item string) bool {
extension := filepath.Ext(item)
if len(r.extensions) > 0 {
if _, ok := r.extensions[extension]; ok {
if len(request.extensions) > 0 {
if _, ok := request.extensions[extension]; ok {
return true
} else if !r.allExtensions {
} else if !request.allExtensions {
return false
}
}
if _, ok := r.extensionDenylist[extension]; ok {
if _, ok := request.extensionDenylist[extension]; ok {
gologger.Verbose().Msgf("Ignoring path %s due to denylist item %s\n", item, extension)
return false
}

76
v2/pkg/protocols/file/operators.go
View File

@ -6,14 +6,16 @@ import (
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
// Match matches a generic data response again a given matcher
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool {
func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
partString := matcher.Part
switch partString {
case "body", "all", "data", "":
@ -22,27 +24,27 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher)
item, ok := data[partString]
if !ok {
return false
return false, []string{}
}
itemStr := types.ToString(item)
switch matcher.GetType() {
case matchers.SizeMatcher:
return matcher.Result(matcher.MatchSize(len(itemStr)))
return matcher.Result(matcher.MatchSize(len(itemStr))), []string{}
case matchers.WordsMatcher:
return matcher.Result(matcher.MatchWords(itemStr, nil))
return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, nil))
case matchers.RegexMatcher:
return matcher.Result(matcher.MatchRegex(itemStr))
return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr))
case matchers.BinaryMatcher:
return matcher.Result(matcher.MatchBinary(itemStr))
return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr))
case matchers.DSLMatcher:
return matcher.Result(matcher.MatchDSL(data))
return matcher.Result(matcher.MatchDSL(data)), []string{}
}
return false
return false, []string{}
}
// Extract performs extracting operation for an extractor on model and returns true or false.
func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
partString := extractor.Part
switch partString {
case "body", "all", "data", "":
@ -64,49 +66,27 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext
return nil
}
// responseToDSLMap converts a DNS response to a map for use in DSL matching
func (r *Request) responseToDSLMap(raw, host, matched string) output.InternalEvent {
data := make(output.InternalEvent, 5)
// Some data regarding the request metadata
data["path"] = host
data["matched"] = matched
data["raw"] = raw
data["template-id"] = r.options.TemplateID
data["template-info"] = r.options.TemplateInfo
data["template-path"] = r.options.TemplatePath
return data
// responseToDSLMap converts a file response to a map for use in DSL matching
func (request *Request) responseToDSLMap(raw, inputFilePath, matchedFileName string) output.InternalEvent {
return output.InternalEvent{
"path": inputFilePath,
"matched": matchedFileName,
"raw": raw,
"template-id": request.options.TemplateID,
"template-info": request.options.TemplateInfo,
"template-path": request.options.TemplatePath,
}
}
// MakeResultEvent creates a result event from internal wrapped event
func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
if len(wrapped.OperatorsResult.DynamicValues) > 0 {
return nil
}
results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1)
func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
results := protocols.MakeDefaultResultEvent(request, wrapped)
// If we have multiple matchers with names, write each of them separately.
if len(wrapped.OperatorsResult.Matches) > 0 {
for k := range wrapped.OperatorsResult.Matches {
data := r.makeResultEventItem(wrapped)
data.MatcherName = k
results = append(results, data)
}
} else if len(wrapped.OperatorsResult.Extracts) > 0 {
for k, v := range wrapped.OperatorsResult.Extracts {
data := r.makeResultEventItem(wrapped)
data.ExtractedResults = v
data.ExtractorName = k
results = append(results, data)
}
} else {
data := r.makeResultEventItem(wrapped)
results = append(results, data)
}
raw, ok := wrapped.InternalEvent["raw"]
if !ok {
return results
}
rawStr, ok := raw.(string)
if !ok {
return results
@ -133,7 +113,11 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu
return results
}
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
func (request *Request) GetCompiledOperators() []*operators.Operators {
return []*operators.Operators{request.CompiledOperators}
}
func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),
@ -141,7 +125,7 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out
Type: "file",
Path: types.ToString(wrapped.InternalEvent["path"]),
Matched: types.ToString(wrapped.InternalEvent["matched"]),
Host: types.ToString(wrapped.InternalEvent["matched"]),
Host: types.ToString(wrapped.InternalEvent["host"]),
ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
Response: types.ToString(wrapped.InternalEvent["raw"]),
Timestamp: time.Now(),

98
v2/pkg/protocols/file/operators_test.go
View File

@ -72,8 +72,9 @@ func TestFileOperatorMatch(t *testing.T) {
err = matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid response")
isMatched, matched := request.Match(event, matcher)
require.True(t, isMatched, "could not match valid response")
require.Equal(t, matcher.Words, matched)
})
t.Run("negative", func(t *testing.T) {
@ -86,8 +87,9 @@ func TestFileOperatorMatch(t *testing.T) {
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile negative matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid negative response matcher")
isMatched, matched := request.Match(event, matcher)
require.True(t, isMatched, "could not match valid negative response matcher")
require.Equal(t, []string{}, matched)
})
t.Run("invalid", func(t *testing.T) {
@ -99,8 +101,9 @@ func TestFileOperatorMatch(t *testing.T) {
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.False(t, matched, "could match invalid response matcher")
isMatched, matched := request.Match(event, matcher)
require.False(t, isMatched, "could match invalid response matcher")
require.Equal(t, []string{}, matched)
})
}
@ -156,7 +159,62 @@ func TestFileOperatorExtract(t *testing.T) {
})
}
func TestFileMakeResult(t *testing.T) {
func TestFileMakeResultWithOrMatcher(t *testing.T) {
expectedValue := []string{"1.1.1.1"}
namedMatcherName := "test"
finalEvent := testFileMakeResultOperators(t, "or")
require.Equal(t, namedMatcherName, finalEvent.Results[0].MatcherName)
require.Equal(t, expectedValue, finalEvent.OperatorsResult.Matches[namedMatcherName], "could not get matched value")
}
func TestFileMakeResultWithAndMatcher(t *testing.T) {
finalEvent := testFileMakeResultOperators(t, "and")
require.Equal(t, "", finalEvent.Results[0].MatcherName)
require.Empty(t, finalEvent.OperatorsResult.Matches)
}
func testFileMakeResultOperators(t *testing.T, matcherCondition string) *output.InternalWrappedEvent {
expectedValue := []string{"1.1.1.1"}
namedMatcherName := "test"
matcher := []*matchers.Matcher{
{
Part: "raw",
Type: "word",
Words: expectedValue,
},
{
Name: namedMatcherName,
Part: "raw",
Type: "word",
Words: expectedValue,
},
}
expectedValues := map[string][]string{
"word-1": expectedValue,
namedMatcherName: expectedValue,
}
finalEvent := testFileMakeResult(t, matcher, matcherCondition, true)
for matcherName, matchedValues := range expectedValues {
var matchesOne = false
for i := 0; i <= len(expectedValue); i++ {
resultEvent := finalEvent.Results[i]
if matcherName == resultEvent.MatcherName {
matchesOne = true
}
}
require.True(t, matchesOne)
require.Equal(t, matchedValues, finalEvent.OperatorsResult.Matches[matcherName], "could not get matched value")
}
finalEvent = testFileMakeResult(t, matcher, matcherCondition, false)
require.Equal(t, 1, len(finalEvent.Results))
return finalEvent
}
func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondition string, isDebug bool) *output.InternalWrappedEvent {
options := testutils.DefaultOptions
testutils.Init(options)
@ -168,12 +226,8 @@ func TestFileMakeResult(t *testing.T) {
Extensions: []string{"*", ".lock"},
ExtensionDenylist: []string{".go"},
Operators: operators.Operators{
Matchers: []*matchers.Matcher{{
Name: "test",
Part: "raw",
Type: "word",
Words: []string{"1.1.1.1"},
}},
MatchersCondition: matcherCondition,
Matchers: matchers,
Extractors: []*extractors.Extractor{{
Part: "raw",
Type: "regex",
@ -188,20 +242,24 @@ func TestFileMakeResult(t *testing.T) {
err := request.Compile(executerOpts)
require.Nil(t, err, "could not compile file request")
resp := "test-data\r\n1.1.1.1\r\n"
event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one")
matchedFileName := "test.txt"
fileContent := "test-data\r\n1.1.1.1\r\n"
event := request.responseToDSLMap(fileContent, "/tmp", matchedFileName)
require.Len(t, event, 6, "could not get correct number of items in dsl map")
require.Equal(t, resp, event["raw"], "could not get correct resp")
require.Equal(t, fileContent, event["raw"], "could not get correct resp")
finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
if request.CompiledOperators != nil {
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract)
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, isDebug)
if ok && result != nil {
finalEvent.OperatorsResult = result
finalEvent.Results = request.MakeResultEvent(finalEvent)
}
}
require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results")
require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results")
require.Equal(t, "1.1.1.1", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results")
resultEvent := finalEvent.Results[0]
require.Equal(t, "1.1.1.1", resultEvent.ExtractedResults[0], "could not get correct extracted results")
require.Equal(t, matchedFileName, resultEvent.Matched, "could not get matched value")
return finalEvent
}

53
v2/pkg/protocols/file/request.go
View File

@ -5,75 +5,74 @@ import (
"os"
"github.com/pkg/errors"
"github.com/remeh/sizedwaitgroup"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
"github.com/remeh/sizedwaitgroup"
)
var _ protocols.Request = &Request{}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
wg := sizedwaitgroup.New(r.options.Options.BulkSize)
func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
wg := sizedwaitgroup.New(request.options.Options.BulkSize)
err := r.getInputPaths(input, func(data string) {
err := request.getInputPaths(input, func(data string) {
wg.Add()
go func(data string) {
go func(filePath string) {
defer wg.Done()
file, err := os.Open(data)
file, err := os.Open(filePath)
if err != nil {
gologger.Error().Msgf("Could not open file path %s: %s\n", data, err)
gologger.Error().Msgf("Could not open file path %s: %s\n", filePath, err)
return
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
gologger.Error().Msgf("Could not stat file path %s: %s\n", data, err)
gologger.Error().Msgf("Could not stat file path %s: %s\n", filePath, err)
return
}
if stat.Size() >= int64(r.MaxSize) {
gologger.Verbose().Msgf("Could not process path %s: exceeded max size\n", data)
if stat.Size() >= int64(request.MaxSize) {
gologger.Verbose().Msgf("Could not process path %s: exceeded max size\n", filePath)
return
}
buffer, err := ioutil.ReadAll(file)
if err != nil {
gologger.Error().Msgf("Could not read file path %s: %s\n", data, err)
gologger.Error().Msgf("Could not read file path %s: %s\n", filePath, err)
return
}
dataStr := tostring.UnsafeToString(buffer)
if r.options.Options.Debug || r.options.Options.DebugRequests {
gologger.Info().Msgf("[%s] Dumped file request for %s", r.options.TemplateID, data)
gologger.Print().Msgf("%s", dataStr)
}
gologger.Verbose().Msgf("[%s] Sent FILE request to %s", r.options.TemplateID, data)
outputEvent := r.responseToDSLMap(dataStr, input, data)
gologger.Verbose().Msgf("[%s] Sent FILE request to %s", request.options.TemplateID, filePath)
outputEvent := request.responseToDSLMap(dataStr, input, filePath)
for k, v := range previous {
outputEvent[k] = v
}
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if r.CompiledOperators != nil {
result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract)
if ok && result != nil {
event.OperatorsResult = result
event.Results = r.MakeResultEvent(event)
}
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Info().Msgf("[%s] Dumped file request for %s", request.options.TemplateID, filePath)
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, dataStr, request.options.Options.NoColor))
}
callback(event)
}(data)
})
wg.Wait()
if err != nil {
r.options.Output.Request(r.options.TemplateID, input, "file", err)
r.options.Progress.IncrementFailedRequestsBy(1)
request.options.Output.Request(request.options.TemplateID, input, "file", err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not send file request")
}
r.options.Progress.IncrementRequests()
request.options.Progress.IncrementRequests()
return nil
}

16
v2/pkg/protocols/headless/headless.go
View File

@ -32,24 +32,24 @@ type Step struct {
}
// GetID returns the unique ID of the request if any.
func (r *Request) GetID() string {
return r.ID
func (request *Request) GetID() string {
return request.ID
}
// Compile compiles the protocol request for further execution.
func (r *Request) Compile(options *protocols.ExecuterOptions) error {
if len(r.Matchers) > 0 || len(r.Extractors) > 0 {
compiled := &r.Operators
func (request *Request) Compile(options *protocols.ExecuterOptions) error {
if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
compiled := &request.Operators
if err := compiled.Compile(); err != nil {
return errors.Wrap(err, "could not compile operators")
}
r.CompiledOperators = compiled
request.CompiledOperators = compiled
}
r.options = options
request.options = options
return nil
}
// Requests returns the total number of requests the YAML rule will perform
func (r *Request) Requests() int {
func (request *Request) Requests() int {
return 1
}

77
v2/pkg/protocols/headless/operators.go
View File

@ -4,14 +4,16 @@ import (
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
// Match matches a generic data response again a given matcher
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool {
func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
partString := matcher.Part
switch partString {
case "body", "resp", "":
@ -20,27 +22,27 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher)
item, ok := data[partString]
if !ok {
return false
return false, []string{}
}
itemStr := types.ToString(item)
switch matcher.GetType() {
case matchers.SizeMatcher:
return matcher.Result(matcher.MatchSize(len(itemStr)))
return matcher.Result(matcher.MatchSize(len(itemStr))), []string{}
case matchers.WordsMatcher:
return matcher.Result(matcher.MatchWords(itemStr, nil))
return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, nil))
case matchers.RegexMatcher:
return matcher.Result(matcher.MatchRegex(itemStr))
return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr))
case matchers.BinaryMatcher:
return matcher.Result(matcher.MatchBinary(itemStr))
return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr))
case matchers.DSLMatcher:
return matcher.Result(matcher.MatchDSL(data))
return matcher.Result(matcher.MatchDSL(data)), []string{}
}
return false
return false, []string{}
}
// Extract performs extracting operation for an extractor on model and returns true or false.
func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
partString := extractor.Part
switch partString {
case "body", "resp", "":
@ -62,50 +64,29 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext
return nil
}
// responseToDSLMap converts a DNS response to a map for use in DSL matching
func (r *Request) responseToDSLMap(resp, req, host, matched string) output.InternalEvent {
data := make(output.InternalEvent, 5)
// Some data regarding the request metadata
data["host"] = host
data["matched"] = matched
data["req"] = req
data["data"] = resp
data["template-id"] = r.options.TemplateID
data["template-info"] = r.options.TemplateInfo
data["template-path"] = r.options.TemplatePath
return data
// responseToDSLMap converts a headless response to a map for use in DSL matching
func (request *Request) responseToDSLMap(resp, req, host, matched string) output.InternalEvent {
return output.InternalEvent{
"host": host,
"matched": matched,
"req": req,
"data": resp,
"template-id": request.options.TemplateID,
"template-info": request.options.TemplateInfo,
"template-path": request.options.TemplatePath,
}
}
// MakeResultEvent creates a result event from internal wrapped event
func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
if len(wrapped.OperatorsResult.DynamicValues) > 0 {
return nil
}
results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1)
// If we have multiple matchers with names, write each of them separately.
if len(wrapped.OperatorsResult.Matches) > 0 {
for k := range wrapped.OperatorsResult.Matches {
data := r.makeResultEventItem(wrapped)
data.MatcherName = k
results = append(results, data)
}
} else if len(wrapped.OperatorsResult.Extracts) > 0 {
for k, v := range wrapped.OperatorsResult.Extracts {
data := r.makeResultEventItem(wrapped)
data.ExtractedResults = v
data.ExtractorName = k
results = append(results, data)
}
} else {
data := r.makeResultEventItem(wrapped)
results = append(results, data)
}
return results
func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
return protocols.MakeDefaultResultEvent(request, wrapped)
}
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
func (request *Request) GetCompiledOperators() []*operators.Operators {
return []*operators.Operators{request.CompiledOperators}
}
func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),

53
v2/pkg/protocols/headless/request.go
View File

@ -6,75 +6,72 @@ import (
"time"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
)
var _ protocols.Request = &Request{}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error {
instance, err := r.options.Browser.NewInstance()
func (request *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error {
instance, err := request.options.Browser.NewInstance()
if err != nil {
r.options.Output.Request(r.options.TemplateID, input, "headless", err)
r.options.Progress.IncrementFailedRequestsBy(1)
request.options.Output.Request(request.options.TemplateID, input, "headless", err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could get html element")
}
defer instance.Close()
parsed, err := url.Parse(input)
if err != nil {
r.options.Output.Request(r.options.TemplateID, input, "headless", err)
r.options.Progress.IncrementFailedRequestsBy(1)
request.options.Output.Request(request.options.TemplateID, input, "headless", err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could get html element")
}
out, page, err := instance.Run(parsed, r.Steps, time.Duration(r.options.Options.PageTimeout)*time.Second)
out, page, err := instance.Run(parsed, request.Steps, time.Duration(request.options.Options.PageTimeout)*time.Second)
if err != nil {
r.options.Output.Request(r.options.TemplateID, input, "headless", err)
r.options.Progress.IncrementFailedRequestsBy(1)
request.options.Output.Request(request.options.TemplateID, input, "headless", err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could get html element")
}
defer page.Close()
r.options.Output.Request(r.options.TemplateID, input, "headless", nil)
r.options.Progress.IncrementRequests()
request.options.Output.Request(request.options.TemplateID, input, "headless", nil)
request.options.Progress.IncrementRequests()
gologger.Verbose().Msgf("Sent Headless request to %s", input)
reqBuilder := &strings.Builder{}
if r.options.Options.Debug || r.options.Options.DebugRequests {
gologger.Info().Msgf("[%s] Dumped Headless request for %s", r.options.TemplateID, input)
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, input)
for _, act := range r.Steps {
for _, act := range request.Steps {
reqBuilder.WriteString(act.String())
reqBuilder.WriteString("\n")
}
gologger.Print().Msgf("%s", reqBuilder.String())
}
var respBody string
var responseBody string
html, err := page.Page().Element("html")
if err == nil {
respBody, _ = html.HTML()
responseBody, _ = html.HTML()
}
outputEvent := r.responseToDSLMap(respBody, reqBuilder.String(), input, input)
outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), input, input)
for k, v := range out {
outputEvent[k] = v
}
if r.options.Options.Debug || r.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped Headless response for %s", r.options.TemplateID, input)
gologger.Print().Msgf("%s", respBody)
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped Headless response for %s", request.options.TemplateID, input)
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseBody, request.options.Options.NoColor))
}
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if r.CompiledOperators != nil {
result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract)
if ok && result != nil {
event.OperatorsResult = result
event.Results = r.MakeResultEvent(event)
}
}
callback(event)
return nil
}

16
v2/pkg/protocols/http/cluster.go
View File

@ -9,20 +9,20 @@ import (
// This used by the clustering engine to decide whether two requests
// are similar enough to be considered one and can be checked by
// just adding the matcher/extractors for the request and the correct IDs.
func (r *Request) CanCluster(other *Request) bool {
if len(r.Payloads) > 0 || len(r.Raw) > 0 || len(r.Body) > 0 || r.Unsafe || r.ReqCondition || r.Name != "" {
func (request *Request) CanCluster(other *Request) bool {
if len(request.Payloads) > 0 || len(request.Raw) > 0 || len(request.Body) > 0 || request.Unsafe || request.ReqCondition || request.Name != "" {
return false
}
if r.Method != other.Method ||
r.MaxRedirects != other.MaxRedirects ||
r.CookieReuse != other.CookieReuse ||
r.Redirects != other.Redirects {
if request.Method != other.Method ||
request.MaxRedirects != other.MaxRedirects ||
request.CookieReuse != other.CookieReuse ||
request.Redirects != other.Redirects {
return false
}
if !compare.StringSlice(r.Path, other.Path) {
if !compare.StringSlice(request.Path, other.Path) {
return false
}
if !compare.StringMap(r.Headers, other.Headers) {
if !compare.StringMap(request.Headers, other.Headers) {
return false
}
return true

92
v2/pkg/protocols/http/http.go
View File

@ -176,21 +176,21 @@ type Request struct {
}
// GetID returns the unique ID of the request if any.
func (r *Request) GetID() string {
return r.ID
func (request *Request) GetID() string {
return request.ID
}
// Compile compiles the protocol request for further execution.
func (r *Request) Compile(options *protocols.ExecuterOptions) error {
func (request *Request) Compile(options *protocols.ExecuterOptions) error {
connectionConfiguration := &httpclientpool.Configuration{
Threads: r.Threads,
MaxRedirects: r.MaxRedirects,
FollowRedirects: r.Redirects,
CookieReuse: r.CookieReuse,
Threads: request.Threads,
MaxRedirects: request.MaxRedirects,
FollowRedirects: request.Redirects,
CookieReuse: request.CookieReuse,
}
// if the headers contain "Connection" we need to disable the automatic keep alive of the standard library
if _, hasConnectionHeader := r.Headers["Connection"]; hasConnectionHeader {
if _, hasConnectionHeader := request.Headers["Connection"]; hasConnectionHeader {
connectionConfiguration.Connection = &httpclientpool.ConnectionConfiguration{DisableKeepAlive: false}
}
@ -198,49 +198,49 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
if err != nil {
return errors.Wrap(err, "could not get dns client")
}
r.customHeaders = make(map[string]string)
r.httpClient = client
r.options = options
for _, option := range r.options.Options.CustomHeaders {
request.customHeaders = make(map[string]string)
request.httpClient = client
request.options = options
for _, option := range request.options.Options.CustomHeaders {
parts := strings.SplitN(option, ":", 2)
if len(parts) != 2 {
continue
}
r.customHeaders[parts[0]] = strings.TrimSpace(parts[1])
request.customHeaders[parts[0]] = strings.TrimSpace(parts[1])
}
if r.Body != "" && !strings.Contains(r.Body, "\r\n") {
r.Body = strings.ReplaceAll(r.Body, "\n", "\r\n")
if request.Body != "" && !strings.Contains(request.Body, "\r\n") {
request.Body = strings.ReplaceAll(request.Body, "\n", "\r\n")
}
if len(r.Raw) > 0 {
for i, raw := range r.Raw {
if len(request.Raw) > 0 {
for i, raw := range request.Raw {
if !strings.Contains(raw, "\r\n") {
r.Raw[i] = strings.ReplaceAll(raw, "\n", "\r\n")
request.Raw[i] = strings.ReplaceAll(raw, "\n", "\r\n")
}
}
r.rawhttpClient = httpclientpool.GetRawHTTP(options.Options)
request.rawhttpClient = httpclientpool.GetRawHTTP(options.Options)
}
if len(r.Matchers) > 0 || len(r.Extractors) > 0 {
compiled := &r.Operators
if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
compiled := &request.Operators
if compileErr := compiled.Compile(); compileErr != nil {
return errors.Wrap(compileErr, "could not compile operators")
}
r.CompiledOperators = compiled
request.CompiledOperators = compiled
}
// Resolve payload paths from vars if they exists
for name, payload := range r.options.Options.VarsPayload() {
for name, payload := range request.options.Options.VarsPayload() {
payloadStr, ok := payload.(string)
// check if inputs contains the payload
var hasPayloadName bool
// search for markers in all request parts
var inputs []string
inputs = append(inputs, r.Method, r.Body)
inputs = append(inputs, r.Raw...)
for k, v := range r.customHeaders {
inputs = append(inputs, request.Method, request.Body)
inputs = append(inputs, request.Raw...)
for k, v := range request.customHeaders {
inputs = append(inputs, fmt.Sprintf("%s: %s", k, v))
}
for k, v := range r.Headers {
for k, v := range request.Headers {
inputs = append(inputs, fmt.Sprintf("%s: %s", k, v))
}
@ -251,57 +251,57 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
}
}
if ok && hasPayloadName && fileutil.FileExists(payloadStr) {
if r.Payloads == nil {
r.Payloads = make(map[string]interface{})
if request.Payloads == nil {
request.Payloads = make(map[string]interface{})
}
r.Payloads[name] = payloadStr
request.Payloads[name] = payloadStr
}
}
if len(r.Payloads) > 0 {
attackType := r.AttackType
if len(request.Payloads) > 0 {
attackType := request.AttackType
if attackType == "" {
attackType = "batteringram"
}
var ok bool
r.attackType, ok = generators.StringToType[attackType]
request.attackType, ok = generators.StringToType[attackType]
if !ok {
return fmt.Errorf("invalid attack type provided: %s", attackType)
}
// Resolve payload paths if they are files.
for name, payload := range r.Payloads {
for name, payload := range request.Payloads {
payloadStr, ok := payload.(string)
if ok {
final, resolveErr := options.Catalog.ResolvePath(payloadStr, options.TemplatePath)
if resolveErr != nil {
return errors.Wrap(resolveErr, "could not read payload file")
}
r.Payloads[name] = final
request.Payloads[name] = final
}
}
r.generator, err = generators.New(r.Payloads, r.attackType, r.options.TemplatePath)
request.generator, err = generators.New(request.Payloads, request.attackType, request.options.TemplatePath)
if err != nil {
return errors.Wrap(err, "could not parse payloads")
}
}
r.options = options
r.totalRequests = r.Requests()
request.options = options
request.totalRequests = request.Requests()
return nil
}
// Requests returns the total number of requests the YAML rule will perform
func (r *Request) Requests() int {
if r.generator != nil {
payloadRequests := r.generator.NewIterator().Total() * len(r.Raw)
func (request *Request) Requests() int {
if request.generator != nil {
payloadRequests := request.generator.NewIterator().Total() * len(request.Raw)
return payloadRequests
}
if len(r.Raw) > 0 {
requests := len(r.Raw)
if requests == 1 && r.RaceNumberRequests != 0 {
requests *= r.RaceNumberRequests
if len(request.Raw) > 0 {
requests := len(request.Raw)
if requests == 1 && request.RaceNumberRequests != 0 {
requests *= request.RaceNumberRequests
}
return requests
}
return len(r.Path)
return len(request.Path)
}

87
v2/pkg/protocols/http/operators.go
View File

@ -6,46 +6,57 @@ import (
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
// Match matches a generic data response again a given matcher
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool {
func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
item, ok := getMatchPart(matcher.Part, data)
if !ok {
return false
return false, []string{}
}
switch matcher.GetType() {
case matchers.StatusMatcher:
statusCode, ok := data["status_code"]
statusCode, ok := getStatusCode(data)
if !ok {
return false
return false, []string{}
}
status, ok := statusCode.(int)
if !ok {
return false
}
return matcher.Result(matcher.MatchStatusCode(status))
return matcher.Result(matcher.MatchStatusCode(statusCode)), []string{responsehighlighter.CreateStatusCodeSnippet(data["response"].(string), statusCode)}
case matchers.SizeMatcher:
return matcher.Result(matcher.MatchSize(len(item)))
return matcher.Result(matcher.MatchSize(len(item))), []string{}
case matchers.WordsMatcher:
return matcher.Result(matcher.MatchWords(item, r.dynamicValues))
return matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, request.dynamicValues))
case matchers.RegexMatcher:
return matcher.Result(matcher.MatchRegex(item))
return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(item))
case matchers.BinaryMatcher:
return matcher.Result(matcher.MatchBinary(item))
return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(item))
case matchers.DSLMatcher:
return matcher.Result(matcher.MatchDSL(data))
return matcher.Result(matcher.MatchDSL(data)), []string{}
}
return false
return false, []string{}
}
func getStatusCode(data map[string]interface{}) (int, bool) {
statusCodeValue, ok := data["status_code"]
if !ok {
return 0, false
}
statusCode, ok := statusCodeValue.(int)
if !ok {
return 0, false
}
return statusCode, true
}
// Extract performs extracting operation for an extractor on model and returns true or false.
func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
item, ok := getMatchPart(extractor.Part, data)
if !ok {
return nil
@ -86,8 +97,8 @@ func getMatchPart(part string, data output.InternalEvent) (string, bool) {
}
// responseToDSLMap converts an HTTP response to a map for use in DSL matching
func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} {
data := make(map[string]interface{}, len(extra)+8+len(resp.Header)+len(resp.Cookies()))
func (request *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) output.InternalEvent {
data := make(output.InternalEvent, 12+len(extra)+len(resp.Header)+len(resp.Cookies()))
for k, v := range extra {
data[k] = v
}
@ -107,42 +118,22 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r
data["content_length"] = resp.ContentLength
data["all_headers"] = headers
data["duration"] = duration.Seconds()
data["template-id"] = r.options.TemplateID
data["template-info"] = r.options.TemplateInfo
data["template-path"] = r.options.TemplatePath
data["template-id"] = request.options.TemplateID
data["template-info"] = request.options.TemplateInfo
data["template-path"] = request.options.TemplatePath
return data
}
// MakeResultEvent creates a result event from internal wrapped event
func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
if len(wrapped.OperatorsResult.DynamicValues) > 0 && !wrapped.OperatorsResult.Matched {
return nil
}
results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1)
// If we have multiple matchers with names, write each of them separately.
if len(wrapped.OperatorsResult.Matches) > 0 {
for k := range wrapped.OperatorsResult.Matches {
data := r.makeResultEventItem(wrapped)
data.MatcherName = k
results = append(results, data)
}
} else if len(wrapped.OperatorsResult.Extracts) > 0 {
for k, v := range wrapped.OperatorsResult.Extracts {
data := r.makeResultEventItem(wrapped)
data.ExtractedResults = v
data.ExtractorName = k
results = append(results, data)
}
} else {
data := r.makeResultEventItem(wrapped)
results = append(results, data)
}
return results
func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
return protocols.MakeDefaultResultEvent(request, wrapped)
}
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
func (request *Request) GetCompiledOperators() []*operators.Operators {
return []*operators.Operators{request.CompiledOperators}
}
func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),

17
v2/pkg/protocols/http/operators_test.go
View File

@ -84,8 +84,9 @@ func TestHTTPOperatorMatch(t *testing.T) {
err = matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid response")
isMatched, matched := request.Match(event, matcher)
require.True(t, isMatched, "could not match valid response")
require.Equal(t, matcher.Words, matched)
})
t.Run("negative", func(t *testing.T) {
@ -98,8 +99,9 @@ func TestHTTPOperatorMatch(t *testing.T) {
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile negative matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid negative response matcher")
isMatched, matched := request.Match(event, matcher)
require.True(t, isMatched, "could not match valid negative response matcher")
require.Equal(t, []string{}, matched)
})
t.Run("invalid", func(t *testing.T) {
@ -111,8 +113,9 @@ func TestHTTPOperatorMatch(t *testing.T) {
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.False(t, matched, "could match invalid response matcher")
isMatched, matched := request.Match(event, matcher)
require.False(t, isMatched, "could match invalid response matcher")
require.Equal(t, []string{}, matched)
})
}
@ -259,7 +262,7 @@ func TestHTTPMakeResult(t *testing.T) {
event["ip"] = "192.169.1.1"
finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
if request.CompiledOperators != nil {
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract)
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, false)
if ok && result != nil {
finalEvent.OperatorsResult = result
finalEvent.Results = request.MakeResultEvent(finalEvent)

251
v2/pkg/protocols/http/request.go
View File

@ -21,6 +21,8 @@ import (
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool"
@ -31,52 +33,52 @@ import (
const defaultMaxWorkers = 150
// executeRaceRequest executes race condition request for a URL
func (r *Request) executeRaceRequest(reqURL string, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
var requests []*generatedRequest
func (request *Request) executeRaceRequest(reqURL string, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
var generatedRequests []*generatedRequest
// Requests within race condition should be dumped once and the output prefilled to allow DSL language to work
// This will introduce a delay and will populate in hacky way the field "request" of outputEvent
generator := r.newGenerator()
generator := request.newGenerator()
requestForDump, err := generator.Make(reqURL, nil, "")
if err != nil {
return err
}
r.setCustomHeaders(requestForDump)
request.setCustomHeaders(requestForDump)
dumpedRequest, err := dump(requestForDump, reqURL)
if err != nil {
return err
}
if r.options.Options.Debug || r.options.Options.DebugRequests {
gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL)
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL)
gologger.Print().Msgf("%s", string(dumpedRequest))
}
previous["request"] = string(dumpedRequest)
// Pre-Generate requests
for i := 0; i < r.RaceNumberRequests; i++ {
generator := r.newGenerator()
request, err := generator.Make(reqURL, nil, "")
for i := 0; i < request.RaceNumberRequests; i++ {
generator := request.newGenerator()
generatedRequest, err := generator.Make(reqURL, nil, "")
if err != nil {
return err
}
requests = append(requests, request)
generatedRequests = append(generatedRequests, generatedRequest)
}
wg := sync.WaitGroup{}
var requestErr error
mutex := &sync.Mutex{}
for i := 0; i < r.RaceNumberRequests; i++ {
for i := 0; i < request.RaceNumberRequests; i++ {
wg.Add(1)
go func(httpRequest *generatedRequest) {
defer wg.Done()
err := r.executeRequest(reqURL, httpRequest, previous, false, callback, 0)
err := request.executeRequest(reqURL, httpRequest, previous, false, callback, 0)
mutex.Lock()
if err != nil {
requestErr = multierr.Append(requestErr, err)
}
mutex.Unlock()
}(requests[i])
r.options.Progress.IncrementRequests()
}(generatedRequests[i])
request.options.Progress.IncrementRequests()
}
wg.Wait()
@ -84,47 +86,47 @@ func (r *Request) executeRaceRequest(reqURL string, previous output.InternalEven
}
// executeRaceRequest executes parallel requests for a template
func (r *Request) executeParallelHTTP(reqURL string, dynamicValues output.InternalEvent, callback protocols.OutputEventCallback) error {
generator := r.newGenerator()
func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output.InternalEvent, callback protocols.OutputEventCallback) error {
generator := request.newGenerator()
// Workers that keeps enqueuing new requests
maxWorkers := r.Threads
maxWorkers := request.Threads
swg := sizedwaitgroup.New(maxWorkers)
var requestErr error
mutex := &sync.Mutex{}
for {
request, err := generator.Make(reqURL, dynamicValues, "")
generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, "")
if err == io.EOF {
break
}
if err != nil {
r.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
return err
}
swg.Add()
go func(httpRequest *generatedRequest) {
defer swg.Done()
r.options.RateLimiter.Take()
request.options.RateLimiter.Take()
previous := make(map[string]interface{})
err := r.executeRequest(reqURL, httpRequest, previous, false, callback, 0)
err := request.executeRequest(reqURL, httpRequest, previous, false, callback, 0)
mutex.Lock()
if err != nil {
requestErr = multierr.Append(requestErr, err)
}
mutex.Unlock()
}(request)
r.options.Progress.IncrementRequests()
}(generatedHttpRequest)
request.options.Progress.IncrementRequests()
}
swg.Wait()
return requestErr
}
// executeTurboHTTP executes turbo http request for a URL
func (r *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
generator := r.newGenerator()
func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
generator := request.newGenerator()
// need to extract the target from the url
URL, err := url.Parse(reqURL)
@ -135,13 +137,13 @@ func (r *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output
pipeOptions := rawhttp.DefaultPipelineOptions
pipeOptions.Host = URL.Host
pipeOptions.MaxConnections = 1
if r.PipelineConcurrentConnections > 0 {
pipeOptions.MaxConnections = r.PipelineConcurrentConnections
if request.PipelineConcurrentConnections > 0 {
pipeOptions.MaxConnections = request.PipelineConcurrentConnections
}
if r.PipelineRequestsPerConnection > 0 {
pipeOptions.MaxPendingRequests = r.PipelineRequestsPerConnection
if request.PipelineRequestsPerConnection > 0 {
pipeOptions.MaxPendingRequests = request.PipelineRequestsPerConnection
}
pipeclient := rawhttp.NewPipelineClient(pipeOptions)
pipeClient := rawhttp.NewPipelineClient(pipeOptions)
// defaultMaxWorkers should be a sufficient value to keep queues always full
maxWorkers := defaultMaxWorkers
@ -154,90 +156,90 @@ func (r *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output
var requestErr error
mutex := &sync.Mutex{}
for {
request, err := generator.Make(reqURL, dynamicValues, "")
generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, "")
if err == io.EOF {
break
}
if err != nil {
r.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
return err
}
request.pipelinedClient = pipeclient
generatedHttpRequest.pipelinedClient = pipeClient
swg.Add()
go func(httpRequest *generatedRequest) {
defer swg.Done()
err := r.executeRequest(reqURL, httpRequest, previous, false, callback, 0)
err := request.executeRequest(reqURL, httpRequest, previous, false, callback, 0)
mutex.Lock()
if err != nil {
requestErr = multierr.Append(requestErr, err)
}
mutex.Unlock()
}(request)
r.options.Progress.IncrementRequests()
}(generatedHttpRequest)
request.options.Progress.IncrementRequests()
}
swg.Wait()
return requestErr
}
// ExecuteWithResults executes the final request on a URL
func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
// verify if pipeline was requested
if r.Pipeline {
return r.executeTurboHTTP(reqURL, dynamicValues, previous, callback)
if request.Pipeline {
return request.executeTurboHTTP(reqURL, dynamicValues, previous, callback)
}
// verify if a basic race condition was requested
if r.Race && r.RaceNumberRequests > 0 {
return r.executeRaceRequest(reqURL, previous, callback)
if request.Race && request.RaceNumberRequests > 0 {
return request.executeRaceRequest(reqURL, previous, callback)
}
// verify if parallel elaboration was requested
if r.Threads > 0 {
return r.executeParallelHTTP(reqURL, dynamicValues, callback)
if request.Threads > 0 {
return request.executeParallelHTTP(reqURL, dynamicValues, callback)
}
generator := r.newGenerator()
generator := request.newGenerator()
requestCount := 1
var requestErr error
for {
hasInteractMarkers := interactsh.HasMatchers(r.CompiledOperators)
hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators)
var interactURL string
if r.options.Interactsh != nil && hasInteractMarkers {
interactURL = r.options.Interactsh.URL()
if request.options.Interactsh != nil && hasInteractMarkers {
interactURL = request.options.Interactsh.URL()
}
request, err := generator.Make(reqURL, dynamicValues, interactURL)
generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, interactURL)
if err == io.EOF {
break
}
if err != nil {
r.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
return err
}
r.dynamicValues = request.dynamicValues
request.dynamicValues = generatedHttpRequest.dynamicValues
// Check if hosts just keep erroring
if r.options.HostErrorsCache != nil && r.options.HostErrorsCache.Check(reqURL) {
if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(reqURL) {
break
}
var gotOutput bool
r.options.RateLimiter.Take()
err = r.executeRequest(reqURL, request, previous, hasInteractMarkers, func(event *output.InternalWrappedEvent) {
request.options.RateLimiter.Take()
err = request.executeRequest(reqURL, generatedHttpRequest, previous, hasInteractMarkers, func(event *output.InternalWrappedEvent) {
// Add the extracts to the dynamic values if any.
if event.OperatorsResult != nil {
gotOutput = true
dynamicValues = generators.MergeMaps(dynamicValues, event.OperatorsResult.DynamicValues)
}
if hasInteractMarkers && r.options.Interactsh != nil {
r.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{
MakeResultFunc: r.MakeResultEvent,
if hasInteractMarkers && request.options.Interactsh != nil {
request.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{
MakeResultFunc: request.MakeResultEvent,
Event: event,
Operators: r.CompiledOperators,
MatchFunc: r.Match,
ExtractFunc: r.Extract,
Operators: request.CompiledOperators,
MatchFunc: request.Match,
ExtractFunc: request.Extract,
})
} else {
callback(event)
@ -248,16 +250,16 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous outp
break
}
if err != nil {
if r.options.HostErrorsCache != nil && r.options.HostErrorsCache.CheckError(err) {
r.options.HostErrorsCache.MarkFailed(reqURL)
if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.CheckError(err) {
request.options.HostErrorsCache.MarkFailed(reqURL)
}
requestErr = err
}
requestCount++
r.options.Progress.IncrementRequests()
request.options.Progress.IncrementRequests()
// If this was a match and we want to stop at first match, skip all further requests.
if (request.original.options.Options.StopAtFirstMatch || r.StopAtFirstMatch) && gotOutput {
if (generatedHttpRequest.original.options.Options.StopAtFirstMatch || request.StopAtFirstMatch) && gotOutput {
break
}
}
@ -269,33 +271,33 @@ const drainReqSize = int64(8 * 1024)
var errStopExecution = errors.New("stop execution due to unresolved variables")
// executeRequest executes the actual generated request and returns error if occurred
func (r *Request) executeRequest(reqURL string, request *generatedRequest, previous output.InternalEvent, hasInteractMarkers bool, callback protocols.OutputEventCallback, requestCount int) error {
r.setCustomHeaders(request)
func (request *Request) executeRequest(reqURL string, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMarkers bool, callback protocols.OutputEventCallback, requestCount int) error {
request.setCustomHeaders(generatedRequest)
var (
resp *http.Response
fromcache bool
fromCache bool
dumpedRequest []byte
err error
)
// For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function
if !request.original.Race {
if !generatedRequest.original.Race {
var dumpError error
dumpedRequest, dumpError = dump(request, reqURL)
dumpedRequest, dumpError = dump(generatedRequest, reqURL)
if dumpError != nil {
return dumpError
}
dumpedRequestString := string(dumpedRequest)
// Check if are there any unresolved variables. If yes, skip unless overriden by user.
if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !r.SkipVariablesCheck {
gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", r.options.TemplateID, reqURL, varErr)
if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck {
gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr)
return errStopExecution
}
if r.options.Options.Debug || r.options.Options.DebugRequests {
gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL)
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL)
gologger.Print().Msgf("%s", dumpedRequestString)
}
}
@ -303,61 +305,61 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ
var formedURL string
var hostname string
timeStart := time.Now()
if request.original.Pipeline {
if request.rawRequest != nil {
formedURL = request.rawRequest.FullURL
if generatedRequest.original.Pipeline {
if generatedRequest.rawRequest != nil {
formedURL = generatedRequest.rawRequest.FullURL
if parsed, parseErr := url.Parse(formedURL); parseErr == nil {
hostname = parsed.Host
}
resp, err = request.pipelinedClient.DoRaw(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data)))
} else if request.request != nil {
resp, err = request.pipelinedClient.Dor(request.request)
resp, err = generatedRequest.pipelinedClient.DoRaw(generatedRequest.rawRequest.Method, reqURL, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)))
} else if generatedRequest.request != nil {
resp, err = generatedRequest.pipelinedClient.Dor(generatedRequest.request)
}
} else if request.original.Unsafe && request.rawRequest != nil {
formedURL = request.rawRequest.FullURL
} else if generatedRequest.original.Unsafe && generatedRequest.rawRequest != nil {
formedURL = generatedRequest.rawRequest.FullURL
if parsed, parseErr := url.Parse(formedURL); parseErr == nil {
hostname = parsed.Host
}
options := request.original.rawhttpClient.Options
options.FollowRedirects = r.Redirects
options.CustomRawBytes = request.rawRequest.UnsafeRawBytes
resp, err = request.original.rawhttpClient.DoRawWithOptions(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data)), options)
options := generatedRequest.original.rawhttpClient.Options
options.FollowRedirects = request.Redirects
options.CustomRawBytes = generatedRequest.rawRequest.UnsafeRawBytes
resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, reqURL, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), options)
} else {
hostname = request.request.URL.Host
formedURL = request.request.URL.String()
hostname = generatedRequest.request.URL.Host
formedURL = generatedRequest.request.URL.String()
// if nuclei-project is available check if the request was already sent previously
if r.options.ProjectFile != nil {
if request.options.ProjectFile != nil {
// if unavailable fail silently
fromcache = true
resp, err = r.options.ProjectFile.Get(dumpedRequest)
fromCache = true
resp, err = request.options.ProjectFile.Get(dumpedRequest)
if err != nil {
fromcache = false
fromCache = false
}
}
if resp == nil {
resp, err = r.httpClient.Do(request.request)
resp, err = request.httpClient.Do(generatedRequest.request)
}
}
if err != nil {
// rawhttp doesn't support draining response bodies.
if resp != nil && resp.Body != nil && request.rawRequest == nil {
if resp != nil && resp.Body != nil && generatedRequest.rawRequest == nil {
_, _ = io.CopyN(ioutil.Discard, resp.Body, drainReqSize)
resp.Body.Close()
}
r.options.Output.Request(r.options.TemplateID, formedURL, "http", err)
r.options.Progress.IncrementErrorsBy(1)
request.options.Output.Request(request.options.TemplateID, formedURL, "http", err)
request.options.Progress.IncrementErrorsBy(1)
// If we have interactsh markers and request times out, still send
// a callback event so in case we receive an interaction, correlation is possible.
if hasInteractMarkers {
outputEvent := r.responseToDSLMap(&http.Response{}, reqURL, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, request.meta)
outputEvent := request.responseToDSLMap(&http.Response{}, reqURL, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, generatedRequest.meta)
if i := strings.LastIndex(hostname, ":"); i != -1 {
hostname = hostname[:i]
}
outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if r.CompiledOperators != nil {
if request.CompiledOperators != nil {
event.InternalEvent = outputEvent
}
callback(event)
@ -371,8 +373,8 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ
resp.Body.Close()
}()
gologger.Verbose().Msgf("[%s] Sent HTTP request to %s", r.options.TemplateID, formedURL)
r.options.Output.Request(r.options.TemplateID, formedURL, "http", err)
gologger.Verbose().Msgf("[%s] Sent HTTP request to %s", request.options.TemplateID, formedURL)
request.options.Output.Request(request.options.TemplateID, formedURL, "http", err)
duration := time.Since(timeStart)
@ -385,8 +387,8 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ
// If the status code is HTTP 101, we should not proceed with reading body.
if resp.StatusCode != http.StatusSwitchingProtocols {
var bodyReader io.Reader
if r.MaxSize != 0 {
bodyReader = io.LimitReader(resp.Body, int64(r.MaxSize))
if request.MaxSize != 0 {
bodyReader = io.LimitReader(resp.Body, int64(request.MaxSize))
} else {
bodyReader = resp.Body
}
@ -394,7 +396,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ
if err != nil {
// Ignore body read due to server misconfiguration errors
if stringsutil.ContainsAny(err.Error(), "gzip: invalid header") {
gologger.Warning().Msgf("[%s] Server sent an invalid gzip header and it was not possible to read the uncompressed body for %s: %s", r.options.TemplateID, formedURL, err.Error())
gologger.Warning().Msgf("[%s] Server sent an invalid gzip header and it was not possible to read the uncompressed body for %s: %s", request.options.TemplateID, formedURL, err.Error())
} else if !stringsutil.ContainsAny(err.Error(), "unexpected EOF", "user canceled") { // ignore EOF and random error
return errors.Wrap(err, "could not read http body")
}
@ -438,66 +440,59 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ
}
}
// Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation)
if r.options.Options.Debug || r.options.Options.DebugResponse {
gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", r.options.TemplateID, formedURL)
gologger.Print().Msgf("%s", string(redirectedResponse))
}
// if nuclei-project is enabled store the response if not previously done
if r.options.ProjectFile != nil && !fromcache {
if err := r.options.ProjectFile.Set(dumpedRequest, resp, data); err != nil {
if request.options.ProjectFile != nil && !fromCache {
if err := request.options.ProjectFile.Set(dumpedRequest, resp, data); err != nil {
return errors.Wrap(err, "could not store in project file")
}
}
matchedURL := reqURL
if request.rawRequest != nil && request.rawRequest.FullURL != "" {
matchedURL = request.rawRequest.FullURL
if generatedRequest.rawRequest != nil && generatedRequest.rawRequest.FullURL != "" {
matchedURL = generatedRequest.rawRequest.FullURL
}
if request.request != nil {
matchedURL = request.request.URL.String()
if generatedRequest.request != nil {
matchedURL = generatedRequest.request.URL.String()
}
finalEvent := make(output.InternalEvent)
outputEvent := r.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, request.meta)
outputEvent := request.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, generatedRequest.meta)
if i := strings.LastIndex(hostname, ":"); i != -1 {
hostname = hostname[:i]
}
outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname)
outputEvent["redirect-chain"] = tostring.UnsafeToString(redirectedResponse)
for k, v := range previous {
for k, v := range previousEvent {
finalEvent[k] = v
}
for k, v := range outputEvent {
finalEvent[k] = v
}
// Add to history the current request number metadata if asked by the user.
if r.ReqCondition {
if request.ReqCondition {
for k, v := range outputEvent {
key := fmt.Sprintf("%s_%d", k, requestCount)
previous[key] = v
previousEvent[key] = v
finalEvent[key] = v
}
}
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
if r.CompiledOperators != nil {
var ok bool
event.OperatorsResult, ok = r.CompiledOperators.Execute(finalEvent, r.Match, r.Extract)
if ok && event.OperatorsResult != nil {
event.OperatorsResult.PayloadValues = request.meta
event.Results = r.MakeResultEvent(event)
}
event.InternalEvent = outputEvent
event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) {
internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta
})
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", request.options.TemplateID, formedURL)
gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, string(redirectedResponse), request.options.Options.NoColor))
}
callback(event)
return nil
}
// setCustomHeaders sets the custom headers for generated request
func (r *Request) setCustomHeaders(req *generatedRequest) {
for k, v := range r.customHeaders {
func (request *Request) setCustomHeaders(req *generatedRequest) {
for k, v := range request.customHeaders {
if req.rawRequest != nil {
req.rawRequest.Headers[k] = v
} else {

9
v2/pkg/protocols/http/request_generator.go
View File

@ -19,11 +19,11 @@ type requestGenerator struct {
}
// newGenerator creates a new request generator instance
func (r *Request) newGenerator() *requestGenerator {
generator := &requestGenerator{request: r, options: r.options}
func (request *Request) newGenerator() *requestGenerator {
generator := &requestGenerator{request: request, options: request.options}
if len(r.Payloads) > 0 {
generator.payloadIterator = r.generator.NewIterator()
if len(request.Payloads) > 0 {
generator.payloadIterator = request.generator.NewIterator()
}
return generator
}
@ -63,7 +63,6 @@ func (r *requestGenerator) nextValue() (value string, payloads map[string]interf
}
}
if len(r.request.Raw) > 0 && r.currentIndex < len(r.request.Raw) {
if r.payloadIterator != nil {
payload, ok := r.payloadIterator.Value()

50
v2/pkg/protocols/network/network.go
View File

@ -116,17 +116,17 @@ type Input struct {
}
// GetID returns the unique ID of the request if any.
func (r *Request) GetID() string {
return r.ID
func (request *Request) GetID() string {
return request.ID
}
// Compile compiles the protocol request for further execution.
func (r *Request) Compile(options *protocols.ExecuterOptions) error {
func (request *Request) Compile(options *protocols.ExecuterOptions) error {
var shouldUseTLS bool
var err error
r.options = options
for _, address := range r.Address {
request.options = options
for _, address := range request.Address {
// check if the connection should be encrypted
if strings.HasPrefix(address, "tls://") {
shouldUseTLS = true
@ -137,13 +137,13 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
if portErr != nil {
return errors.Wrap(portErr, "could not parse address")
}
r.addresses = append(r.addresses, addressKV{ip: addressHost, port: addressPort, tls: shouldUseTLS})
request.addresses = append(request.addresses, addressKV{ip: addressHost, port: addressPort, tls: shouldUseTLS})
} else {
r.addresses = append(r.addresses, addressKV{ip: address, tls: shouldUseTLS})
request.addresses = append(request.addresses, addressKV{ip: address, tls: shouldUseTLS})
}
}
// Pre-compile any input dsl functions before executing the request.
for _, input := range r.Inputs {
for _, input := range request.Inputs {
if input.Type != "" {
continue
}
@ -153,11 +153,11 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
}
// Resolve payload paths from vars if they exists
for name, payload := range r.options.Options.VarsPayload() {
for name, payload := range request.options.Options.VarsPayload() {
payloadStr, ok := payload.(string)
// check if inputs contains the payload
var hasPayloadName bool
for _, input := range r.Inputs {
for _, input := range request.Inputs {
if input.Type != "" {
continue
}
@ -167,36 +167,36 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
}
}
if ok && hasPayloadName && fileutil.FileExists(payloadStr) {
if r.Payloads == nil {
r.Payloads = make(map[string]interface{})
if request.Payloads == nil {
request.Payloads = make(map[string]interface{})
}
r.Payloads[name] = payloadStr
request.Payloads[name] = payloadStr
}
}
if len(r.Payloads) > 0 {
attackType := r.AttackType
if len(request.Payloads) > 0 {
attackType := request.AttackType
if attackType == "" {
attackType = "batteringram"
}
var ok bool
r.attackType, ok = generators.StringToType[attackType]
request.attackType, ok = generators.StringToType[attackType]
if !ok {
return fmt.Errorf("invalid attack type provided: %s", attackType)
}
// Resolve payload paths if they are files.
for name, payload := range r.Payloads {
for name, payload := range request.Payloads {
payloadStr, ok := payload.(string)
if ok {
final, resolveErr := options.Catalog.ResolvePath(payloadStr, options.TemplatePath)
if resolveErr != nil {
return errors.Wrap(resolveErr, "could not read payload file")
}
r.Payloads[name] = final
request.Payloads[name] = final
}
}
r.generator, err = generators.New(r.Payloads, r.attackType, r.options.TemplatePath)
request.generator, err = generators.New(request.Payloads, request.attackType, request.options.TemplatePath)
if err != nil {
return errors.Wrap(err, "could not parse payloads")
}
@ -207,19 +207,19 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error {
if err != nil {
return errors.Wrap(err, "could not get network client")
}
r.dialer = client
request.dialer = client
if len(r.Matchers) > 0 || len(r.Extractors) > 0 {
compiled := &r.Operators
if len(request.Matchers) > 0 || len(request.Extractors) > 0 {
compiled := &request.Operators
if err := compiled.Compile(); err != nil {
return errors.Wrap(err, "could not compile operators")
}
r.CompiledOperators = compiled
request.CompiledOperators = compiled
}
return nil
}
// Requests returns the total number of requests the YAML rule will perform
func (r *Request) Requests() int {
return len(r.Address)
func (request *Request) Requests() int {
return len(request.Address)
}

79
v2/pkg/protocols/network/operators.go
View File

@ -4,14 +4,16 @@ import (
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
// Match matches a generic data response again a given matcher
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool {
func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
partString := matcher.Part
switch partString {
case "body", "all", "":
@ -20,27 +22,27 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher)
item, ok := data[partString]
if !ok {
return false
return false, []string{}
}
itemStr := types.ToString(item)
switch matcher.GetType() {
case matchers.SizeMatcher:
return matcher.Result(matcher.MatchSize(len(itemStr)))
return matcher.Result(matcher.MatchSize(len(itemStr))), []string{}
case matchers.WordsMatcher:
return matcher.Result(matcher.MatchWords(itemStr, r.dynamicValues))
return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, request.dynamicValues))
case matchers.RegexMatcher:
return matcher.Result(matcher.MatchRegex(itemStr))
return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr))
case matchers.BinaryMatcher:
return matcher.Result(matcher.MatchBinary(itemStr))
return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr))
case matchers.DSLMatcher:
return matcher.Result(matcher.MatchDSL(data))
return matcher.Result(matcher.MatchDSL(data)), []string{}
}
return false
return false, []string{}
}
// Extract performs extracting operation for an extractor on model and returns true or false.
func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
partString := extractor.Part
switch partString {
case "body", "all", "":
@ -62,51 +64,30 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext
return nil
}
// responseToDSLMap converts a DNS response to a map for use in DSL matching
func (r *Request) responseToDSLMap(req, resp, raw, host, matched string) output.InternalEvent {
data := make(output.InternalEvent, 6)
// Some data regarding the request metadata
data["host"] = host
data["matched"] = matched
data["request"] = req
data["data"] = resp // Data is the last bytes read
data["raw"] = raw // Raw is the full transaction data for network
data["template-id"] = r.options.TemplateID
data["template-info"] = r.options.TemplateInfo
data["template-path"] = r.options.TemplatePath
return data
// responseToDSLMap converts a network response to a map for use in DSL matching
func (request *Request) responseToDSLMap(req, resp, raw, host, matched string) output.InternalEvent {
return output.InternalEvent{
"host": host,
"matched": matched,
"request": req,
"data": resp, // Data is the last bytes read
"raw": raw, // Raw is the full transaction data for network
"template-id": request.options.TemplateID,
"template-info": request.options.TemplateInfo,
"template-path": request.options.TemplatePath,
}
}
// MakeResultEvent creates a result event from internal wrapped event
func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
if len(wrapped.OperatorsResult.DynamicValues) > 0 {
return nil
}
results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1)
// If we have multiple matchers with names, write each of them separately.
if len(wrapped.OperatorsResult.Matches) > 0 {
for k := range wrapped.OperatorsResult.Matches {
data := r.makeResultEventItem(wrapped)
data.MatcherName = k
results = append(results, data)
}
} else if len(wrapped.OperatorsResult.Extracts) > 0 {
for k, v := range wrapped.OperatorsResult.Extracts {
data := r.makeResultEventItem(wrapped)
data.ExtractedResults = v
data.ExtractorName = k
results = append(results, data)
}
} else {
data := r.makeResultEventItem(wrapped)
results = append(results, data)
}
return results
func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
return protocols.MakeDefaultResultEvent(request, wrapped)
}
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
func (request *Request) GetCompiledOperators() []*operators.Operators {
return []*operators.Operators{request.CompiledOperators}
}
func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),

17
v2/pkg/protocols/network/operators_test.go
View File

@ -70,8 +70,9 @@ func TestNetworkOperatorMatch(t *testing.T) {
err = matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid response")
isMatched, matched := request.Match(event, matcher)
require.True(t, isMatched, "could not match valid response")
require.Equal(t, matcher.Words, matched)
})
t.Run("negative", func(t *testing.T) {
@ -84,8 +85,9 @@ func TestNetworkOperatorMatch(t *testing.T) {
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile negative matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid negative response matcher")
isMatched, matched := request.Match(event, matcher)
require.True(t, isMatched, "could not match valid negative response matcher")
require.Equal(t, []string{}, matched)
})
t.Run("invalid", func(t *testing.T) {
@ -97,8 +99,9 @@ func TestNetworkOperatorMatch(t *testing.T) {
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.False(t, matched, "could match invalid response matcher")
isMatched, matched := request.Match(event, matcher)
require.False(t, isMatched, "could match invalid response matcher")
require.Equal(t, []string{}, matched)
})
}
@ -189,7 +192,7 @@ func TestNetworkMakeResult(t *testing.T) {
finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
event["ip"] = "192.168.1.1"
if request.CompiledOperators != nil {
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract)
result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, false)
if ok && result != nil {
finalEvent.OperatorsResult = result
finalEvent.Results = request.MakeResultEvent(finalEvent)

125
v2/pkg/protocols/network/request.go
View File

@ -10,11 +10,14 @@ import (
"time"
"github.com/pkg/errors"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer"
)
@ -22,15 +25,15 @@ import (
var _ protocols.Request = &Request{}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
address, err := getAddress(input)
if err != nil {
r.options.Output.Request(r.options.TemplateID, input, "network", err)
r.options.Progress.IncrementFailedRequestsBy(1)
request.options.Output.Request(request.options.TemplateID, input, "network", err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not get address from url")
}
for _, kv := range r.addresses {
for _, kv := range request.addresses {
actualAddress := replacer.Replace(kv.ip, map[string]interface{}{"Hostname": address})
if kv.port != "" {
if strings.Contains(address, ":") {
@ -39,7 +42,7 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused
actualAddress = net.JoinHostPort(actualAddress, kv.port)
}
if err := r.executeAddress(actualAddress, address, input, kv.tls, previous, callback); err != nil {
if err := request.executeAddress(actualAddress, address, input, kv.tls, previous, callback); err != nil {
gologger.Verbose().Label("ERR").Msgf("Could not make network request for %s: %s\n", actualAddress, err)
continue
}
@ -48,71 +51,71 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused
}
// executeAddress executes the request for an address
func (r *Request) executeAddress(actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
func (request *Request) executeAddress(actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
if !strings.Contains(actualAddress, ":") {
err := errors.New("no port provided in network protocol request")
r.options.Output.Request(r.options.TemplateID, address, "network", err)
r.options.Progress.IncrementFailedRequestsBy(1)
request.options.Output.Request(request.options.TemplateID, address, "network", err)
request.options.Progress.IncrementFailedRequestsBy(1)
return err
}
if r.generator != nil {
iterator := r.generator.NewIterator()
if request.generator != nil {
iterator := request.generator.NewIterator()
for {
value, ok := iterator.Value()
if !ok {
break
}
if err := r.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil {
if err := request.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil {
return err
}
}
} else {
value := make(map[string]interface{})
if err := r.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil {
if err := request.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil {
return err
}
}
return nil
}
func (r *Request) executeRequestWithPayloads(actualAddress, address, input string, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
func (request *Request) executeRequestWithPayloads(actualAddress, address, input string, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
var (
hostname string
conn net.Conn
err error
)
r.dynamicValues = generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address})
request.dynamicValues = generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address})
if host, _, splitErr := net.SplitHostPort(actualAddress); splitErr == nil {
hostname = host
}
if shouldUseTLS {
conn, err = r.dialer.DialTLS(context.Background(), "tcp", actualAddress)
conn, err = request.dialer.DialTLS(context.Background(), "tcp", actualAddress)
} else {
conn, err = r.dialer.Dial(context.Background(), "tcp", actualAddress)
conn, err = request.dialer.Dial(context.Background(), "tcp", actualAddress)
}
if err != nil {
r.options.Output.Request(r.options.TemplateID, address, "network", err)
r.options.Progress.IncrementFailedRequestsBy(1)
request.options.Output.Request(request.options.TemplateID, address, "network", err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not connect to server request")
}
defer conn.Close()
_ = conn.SetReadDeadline(time.Now().Add(time.Duration(r.options.Options.Timeout) * time.Second))
_ = conn.SetReadDeadline(time.Now().Add(time.Duration(request.options.Options.Timeout) * time.Second))
hasInteractMarkers := interactsh.HasMatchers(r.CompiledOperators)
hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators)
var interactURL string
if r.options.Interactsh != nil && hasInteractMarkers {
interactURL = r.options.Interactsh.URL()
if request.options.Interactsh != nil && hasInteractMarkers {
interactURL = request.options.Interactsh.URL()
}
responseBuilder := &strings.Builder{}
reqBuilder := &strings.Builder{}
inputEvents := make(map[string]interface{})
for _, input := range r.Inputs {
for _, input := range request.Inputs {
var data []byte
switch input.Type {
@ -120,32 +123,32 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin
data, err = hex.DecodeString(input.Data)
default:
if interactURL != "" {
input.Data = r.options.Interactsh.ReplaceMarkers(input.Data, interactURL)
input.Data = request.options.Interactsh.ReplaceMarkers(input.Data, interactURL)
}
data = []byte(input.Data)
}
if err != nil {
r.options.Output.Request(r.options.TemplateID, address, "network", err)
r.options.Progress.IncrementFailedRequestsBy(1)
request.options.Output.Request(request.options.TemplateID, address, "network", err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not write request to server")
}
reqBuilder.Grow(len(input.Data))
finalData, dataErr := expressions.EvaluateByte(data, payloads)
if dataErr != nil {
r.options.Output.Request(r.options.TemplateID, address, "network", dataErr)
r.options.Progress.IncrementFailedRequestsBy(1)
request.options.Output.Request(request.options.TemplateID, address, "network", dataErr)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(dataErr, "could not evaluate template expressions")
}
reqBuilder.Write(finalData)
if varErr := expressions.ContainsUnresolvedVariables(string(finalData)); varErr != nil {
gologger.Warning().Msgf("[%s] Could not make network request for %s: %v\n", r.options.TemplateID, actualAddress, varErr)
gologger.Warning().Msgf("[%s] Could not make network request for %s: %v\n", request.options.TemplateID, actualAddress, varErr)
return nil
}
if _, err := conn.Write(finalData); err != nil {
r.options.Output.Request(r.options.TemplateID, address, "network", err)
r.options.Progress.IncrementFailedRequestsBy(1)
request.options.Output.Request(request.options.TemplateID, address, "network", err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not write request to server")
}
@ -160,44 +163,40 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin
}
// Run any internal extractors for the request here and add found values to map.
if r.CompiledOperators != nil {
values := r.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{input.Name: bufferStr}, r.Extract)
if request.CompiledOperators != nil {
values := request.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{input.Name: bufferStr}, request.Extract)
for k, v := range values {
payloads[k] = v
}
}
}
}
r.options.Progress.IncrementRequests()
request.options.Progress.IncrementRequests()
if r.options.Options.Debug || r.options.Options.DebugRequests {
if request.options.Options.Debug || request.options.Options.DebugRequests {
requestOutput := reqBuilder.String()
gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s", r.options.TemplateID, actualAddress)
gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s", request.options.TemplateID, actualAddress)
gologger.Print().Msgf("%s\nHex: %s", requestOutput, hex.EncodeToString([]byte(requestOutput)))
}
r.options.Output.Request(r.options.TemplateID, actualAddress, "network", err)
request.options.Output.Request(request.options.TemplateID, actualAddress, "network", err)
gologger.Verbose().Msgf("Sent TCP request to %s", actualAddress)
bufferSize := 1024
if r.ReadSize != 0 {
bufferSize = r.ReadSize
if request.ReadSize != 0 {
bufferSize = request.ReadSize
}
final := make([]byte, bufferSize)
n, err := conn.Read(final)
if err != nil && err != io.EOF {
r.options.Output.Request(r.options.TemplateID, address, "network", err)
request.options.Output.Request(request.options.TemplateID, address, "network", err)
return errors.Wrap(err, "could not read from server")
}
responseBuilder.Write(final[:n])
if r.options.Options.Debug || r.options.Options.DebugResponse {
responseOutput := responseBuilder.String()
gologger.Debug().Msgf("[%s] Dumped Network response for %s", r.options.TemplateID, actualAddress)
gologger.Print().Msgf("%s\nHex: %s", responseOutput, hex.EncodeToString([]byte(responseOutput)))
}
outputEvent := r.responseToDSLMap(reqBuilder.String(), string(final[:n]), responseBuilder.String(), input, actualAddress)
outputEvent["ip"] = r.dialer.GetDialedIP(hostname)
response := responseBuilder.String()
outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input, actualAddress)
outputEvent["ip"] = request.dialer.GetDialedIP(hostname)
for k, v := range previous {
outputEvent[k] = v
}
@ -208,26 +207,28 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin
outputEvent[k] = v
}
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
var event *output.InternalWrappedEvent
if interactURL == "" {
if r.CompiledOperators != nil {
result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract)
if ok && result != nil {
event.OperatorsResult = result
event.OperatorsResult.PayloadValues = payloads
event.Results = r.MakeResultEvent(event)
}
}
event = eventcreator.CreateEventWithAdditionalOptions(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) {
wrappedEvent.OperatorsResult.PayloadValues = payloads
})
callback(event)
} else if r.options.Interactsh != nil {
r.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{
MakeResultFunc: r.MakeResultEvent,
} else if request.options.Interactsh != nil {
event = &output.InternalWrappedEvent{InternalEvent: outputEvent}
request.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{
MakeResultFunc: request.MakeResultEvent,
Event: event,
Operators: r.CompiledOperators,
MatchFunc: r.Match,
ExtractFunc: r.Extract,
Operators: request.CompiledOperators,
MatchFunc: request.Match,
ExtractFunc: request.Extract,
})
}
if request.options.Options.Debug || request.options.Options.DebugResponse {
gologger.Debug().Msgf("[%s] Dumped Network response for %s", request.options.TemplateID, actualAddress)
gologger.Print().Msgf("%s\nHex: %s", response, responsehighlighter.Highlight(event.OperatorsResult, hex.EncodeToString([]byte(response)), request.options.Options.NoColor))
}
return nil
}

14
v2/pkg/protocols/offlinehttp/find.go
View File

@ -12,19 +12,19 @@ import (
// getInputPaths parses the specified input paths and returns a compiled
// list of finished absolute paths to the files evaluating any allowlist, denylist,
// glob, file or folders, etc.
func (r *Request) getInputPaths(target string, callback func(string)) error {
func (request *Request) getInputPaths(target string, callback func(string)) error {
processed := make(map[string]struct{})
// Template input includes a wildcard
if strings.Contains(target, "*") {
if err := r.findGlobPathMatches(target, processed, callback); err != nil {
if err := request.findGlobPathMatches(target, processed, callback); err != nil {
return errors.Wrap(err, "could not find glob matches")
}
return nil
}
// Template input is either a file or a directory
file, err := r.findFileMatches(target, processed, callback)
file, err := request.findFileMatches(target, processed, callback)
if err != nil {
return errors.Wrap(err, "could not find file")
}
@ -34,14 +34,14 @@ func (r *Request) getInputPaths(target string, callback func(string)) error {
// Recursively walk down the Templates directory and run all
// the template file checks
if err := r.findDirectoryMatches(target, processed, callback); err != nil {
if err := request.findDirectoryMatches(target, processed, callback); err != nil {
return errors.Wrap(err, "could not find directory matches")
}
return nil
}
// findGlobPathMatches returns the matched files from a glob path
func (r *Request) findGlobPathMatches(absPath string, processed map[string]struct{}, callback func(string)) error {
func (request *Request) findGlobPathMatches(absPath string, processed map[string]struct{}, callback func(string)) error {
matches, err := filepath.Glob(absPath)
if err != nil {
return errors.Errorf("wildcard found, but unable to glob: %s\n", err)
@ -60,7 +60,7 @@ func (r *Request) findGlobPathMatches(absPath string, processed map[string]struc
// findFileMatches finds if a path is an absolute file. If the path
// is a file, it returns true otherwise false with no errors.
func (r *Request) findFileMatches(absPath string, processed map[string]struct{}, callback func(string)) (bool, error) {
func (request *Request) findFileMatches(absPath string, processed map[string]struct{}, callback func(string)) (bool, error) {
info, err := os.Stat(absPath)
if err != nil {
return false, err
@ -79,7 +79,7 @@ func (r *Request) findFileMatches(absPath string, processed map[string]struct{},
}
// findDirectoryMatches finds matches for templates from a directory
func (r *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error {
func (request *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error {
err := godirwalk.Walk(absPath, &godirwalk.Options{
Unsorted: true,
ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction {

11
v2/pkg/protocols/offlinehttp/offlinehttp.go
View File

@ -2,6 +2,7 @@ package offlinehttp
import (
"github.com/pkg/errors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
)
@ -13,23 +14,23 @@ type Request struct {
}
// GetID returns the unique ID of the request if any.
func (r *Request) GetID() string {
func (request *Request) GetID() string {
return ""
}
// Compile compiles the protocol request for further execution.
func (r *Request) Compile(options *protocols.ExecuterOptions) error {
func (request *Request) Compile(options *protocols.ExecuterOptions) error {
for _, operator := range options.Operators {
if err := operator.Compile(); err != nil {
return errors.Wrap(err, "could not compile operators")
}
r.compiledOperators = append(r.compiledOperators, operator)
request.compiledOperators = append(request.compiledOperators, operator)
}
r.options = options
request.options = options
return nil
}
// Requests returns the total number of requests the YAML rule will perform
func (r *Request) Requests() int {
func (request *Request) Requests() int {
return 1
}

82
v2/pkg/protocols/offlinehttp/operators.go
View File

@ -6,42 +6,57 @@ import (
"time"
"github.com/projectdiscovery/nuclei/v2/pkg/model"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors"
"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
)
// Match matches a generic data response again a given matcher
func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool {
func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) {
item, ok := getMatchPart(matcher.Part, data)
if !ok {
return false
return false, []string{}
}
switch matcher.GetType() {
case matchers.StatusMatcher:
statusCode, ok := data["status_code"]
statusCode, ok := getStatusCode(data)
if !ok {
return false
return false, []string{}
}
return matcher.Result(matcher.MatchStatusCode(statusCode.(int)))
return matcher.Result(matcher.MatchStatusCode(statusCode)), []string{responsehighlighter.CreateStatusCodeSnippet(data["response"].(string), statusCode)}
case matchers.SizeMatcher:
return matcher.Result(matcher.MatchSize(len(item)))
return matcher.Result(matcher.MatchSize(len(item))), []string{}
case matchers.WordsMatcher:
return matcher.Result(matcher.MatchWords(item, nil))
return matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, nil))
case matchers.RegexMatcher:
return matcher.Result(matcher.MatchRegex(item))
return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(item))
case matchers.BinaryMatcher:
return matcher.Result(matcher.MatchBinary(item))
return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(item))
case matchers.DSLMatcher:
return matcher.Result(matcher.MatchDSL(data))
return matcher.Result(matcher.MatchDSL(data)), []string{}
}
return false
return false, []string{}
}
func getStatusCode(data map[string]interface{}) (int, bool) {
statusCodeValue, ok := data["status_code"]
if !ok {
return 0, false
}
statusCode, ok := statusCodeValue.(int)
if !ok {
return 0, false
}
return statusCode, true
}
// Extract performs extracting operation for an extractor on model and returns true or false.
func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} {
item, ok := getMatchPart(extractor.Part, data)
if !ok {
return nil
@ -78,8 +93,8 @@ func getMatchPart(part string, data output.InternalEvent) (string, bool) {
}
// responseToDSLMap converts an HTTP response to a map for use in DSL matching
func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} {
data := make(map[string]interface{}, len(extra)+8+len(resp.Header)+len(resp.Cookies()))
func (request *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) output.InternalEvent {
data := make(output.InternalEvent, 12+len(extra)+len(resp.Header)+len(resp.Cookies()))
for k, v := range extra {
data[k] = v
}
@ -100,41 +115,22 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r
}
data["all_headers"] = headers
data["duration"] = duration.Seconds()
data["template-id"] = r.options.TemplateID
data["template-info"] = r.options.TemplateInfo
data["template-path"] = r.options.TemplatePath
data["template-id"] = request.options.TemplateID
data["template-info"] = request.options.TemplateInfo
data["template-path"] = request.options.TemplatePath
return data
}
// MakeResultEvent creates a result event from internal wrapped event
func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
if len(wrapped.OperatorsResult.DynamicValues) > 0 {
return nil
}
results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1)
// If we have multiple matchers with names, write each of them separately.
if len(wrapped.OperatorsResult.Matches) > 0 {
for k := range wrapped.OperatorsResult.Matches {
data := r.makeResultEventItem(wrapped)
data.MatcherName = k
results = append(results, data)
}
} else if len(wrapped.OperatorsResult.Extracts) > 0 {
for k, v := range wrapped.OperatorsResult.Extracts {
data := r.makeResultEventItem(wrapped)
data.ExtractedResults = v
data.ExtractorName = k
results = append(results, data)
}
} else {
data := r.makeResultEventItem(wrapped)
results = append(results, data)
}
return results
func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
return protocols.MakeDefaultResultEvent(request, wrapped)
}
func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
func (request *Request) GetCompiledOperators() []*operators.Operators {
return request.compiledOperators
}
func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent {
data := &output.ResultEvent{
TemplateID: types.ToString(wrapped.InternalEvent["template-id"]),
TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]),

17
v2/pkg/protocols/offlinehttp/operators_test.go
View File

@ -76,8 +76,9 @@ func TestHTTPOperatorMatch(t *testing.T) {
err = matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid response")
isMatched, matched := request.Match(event, matcher)
require.True(t, isMatched, "could not match valid response")
require.Equal(t, matcher.Words, matched)
})
t.Run("negative", func(t *testing.T) {
@ -90,8 +91,9 @@ func TestHTTPOperatorMatch(t *testing.T) {
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile negative matcher")
matched := request.Match(event, matcher)
require.True(t, matched, "could not match valid negative response matcher")
isMatched, matched := request.Match(event, matcher)
require.True(t, isMatched, "could not match valid negative response matcher")
require.Equal(t, []string{}, matched)
})
t.Run("invalid", func(t *testing.T) {
@ -103,8 +105,9 @@ func TestHTTPOperatorMatch(t *testing.T) {
err := matcher.CompileMatchers()
require.Nil(t, err, "could not compile matcher")
matched := request.Match(event, matcher)
require.False(t, matched, "could match invalid response matcher")
isMatched, matched := request.Match(event, matcher)
require.False(t, isMatched, "could match invalid response matcher")
require.Equal(t, []string{}, matched)
})
}
@ -201,7 +204,7 @@ func TestHTTPMakeResult(t *testing.T) {
event["ip"] = "192.169.1.1"
finalEvent := &output.InternalWrappedEvent{InternalEvent: event}
for _, operator := range request.compiledOperators {
result, ok := operator.Execute(event, request.Match, request.Extract)
result, ok := operator.Execute(event, request.Match, request.Extract, false)
if ok && result != nil {
finalEvent.OperatorsResult = result
finalEvent.Results = request.MakeResultEvent(finalEvent)

34
v2/pkg/protocols/offlinehttp/request.go
View File

@ -8,11 +8,13 @@ import (
"strings"
"github.com/pkg/errors"
"github.com/remeh/sizedwaitgroup"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
"github.com/remeh/sizedwaitgroup"
)
var _ protocols.Request = &Request{}
@ -20,10 +22,10 @@ var _ protocols.Request = &Request{}
const maxSize = 5 * 1024 * 1024
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
wg := sizedwaitgroup.New(r.options.Options.BulkSize)
func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
wg := sizedwaitgroup.New(request.options.Options.BulkSize)
err := r.getInputPaths(input, func(data string) {
err := request.getInputPaths(input, func(data string) {
wg.Add()
go func(data string) {
@ -59,11 +61,11 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused
return
}
if r.options.Options.Debug || r.options.Options.DebugRequests {
gologger.Info().Msgf("[%s] Dumped offline-http request for %s", r.options.TemplateID, data)
if request.options.Options.Debug || request.options.Options.DebugRequests {
gologger.Info().Msgf("[%s] Dumped offline-http request for %s", request.options.TemplateID, data)
gologger.Print().Msgf("%s", dataStr)
}
gologger.Verbose().Msgf("[%s] Sent OFFLINE-HTTP request to %s", r.options.TemplateID, data)
gologger.Verbose().Msgf("[%s] Sent OFFLINE-HTTP request to %s", request.options.TemplateID, data)
dumpedResponse, err := httputil.DumpResponse(resp, true)
if err != nil {
@ -77,31 +79,23 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused
return
}
outputEvent := r.responseToDSLMap(resp, data, data, data, tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(body), headersToString(resp.Header), 0, nil)
outputEvent := request.responseToDSLMap(resp, data, data, data, tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(body), headersToString(resp.Header), 0, nil)
outputEvent["ip"] = ""
for k, v := range previous {
outputEvent[k] = v
}
for _, operator := range r.compiledOperators {
event := &output.InternalWrappedEvent{InternalEvent: outputEvent}
var ok bool
event.OperatorsResult, ok = operator.Execute(outputEvent, r.Match, r.Extract)
if ok && event.OperatorsResult != nil {
event.Results = r.MakeResultEvent(event)
}
event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
callback(event)
}
}(data)
})
wg.Wait()
if err != nil {
r.options.Output.Request(r.options.TemplateID, input, "file", err)
r.options.Progress.IncrementFailedRequestsBy(1)
request.options.Output.Request(request.options.TemplateID, input, "file", err)
request.options.Progress.IncrementFailedRequestsBy(1)
return errors.Wrap(err, "could not send file request")
}
r.options.Progress.IncrementRequests()
request.options.Progress.IncrementRequests()
return nil
}

40
v2/pkg/protocols/protocols.go
View File

@ -74,13 +74,49 @@ type Request interface {
// condition matching. So, two requests can be sent and their match can
// be evaluated from the third request by using the IDs for both requests.
GetID() string
// Match performs matching operation for a matcher on model and returns true or false.
Match(data map[string]interface{}, matcher *matchers.Matcher) bool
// Match performs matching operation for a matcher on model and returns:
// true and a list of matched snippets if the matcher type is supports it
// otherwise false and an empty string slice
Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string)
// Extract performs extracting operation for an extractor on model and returns true or false.
Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{}
// ExecuteWithResults executes the protocol requests and returns results instead of writing them.
ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback OutputEventCallback) error
// MakeResultEventItem creates a result event from internal wrapped event. Intended to be used by MakeResultEventItem internally
MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent
// MakeResultEvent creates a flat list of result events from an internal wrapped event, based on successful matchers and extracted data
MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent
// GetCompiledOperators returns a list of the compiled operators
GetCompiledOperators() []*operators.Operators
}
// OutputEventCallback is a callback event for any results found during scanning.
type OutputEventCallback func(result *output.InternalWrappedEvent)
func MakeDefaultResultEvent(request Request, wrapped *output.InternalWrappedEvent) []*output.ResultEvent {
if len(wrapped.OperatorsResult.DynamicValues) > 0 && !wrapped.OperatorsResult.Matched {
return nil
}
results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1)
// If we have multiple matchers with names, write each of them separately.
if len(wrapped.OperatorsResult.Matches) > 0 {
for matcherNames := range wrapped.OperatorsResult.Matches {
data := request.MakeResultEventItem(wrapped)
data.MatcherName = matcherNames
results = append(results, data)
}
} else if len(wrapped.OperatorsResult.Extracts) > 0 {
for k, v := range wrapped.OperatorsResult.Extracts {
data := request.MakeResultEventItem(wrapped)
data.ExtractorName = k
data.ExtractedResults = v
results = append(results, data)
}
} else {
data := request.MakeResultEventItem(wrapped)
results = append(results, data)
}
return results
}

7
v2/pkg/workflows/execute_test.go
View File

@ -3,12 +3,13 @@ package workflows
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/projectdiscovery/nuclei/v2/pkg/operators"
"github.com/projectdiscovery/nuclei/v2/pkg/output"
"github.com/projectdiscovery/nuclei/v2/pkg/progress"
"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
"github.com/projectdiscovery/nuclei/v2/pkg/types"
"github.com/stretchr/testify/require"
)
func TestWorkflowsSimple(t *testing.T) {
@ -106,7 +107,7 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) {
firstInput = input
}, outputs: []*output.InternalWrappedEvent{
{OperatorsResult: &operators.Result{
Matches: map[string]struct{}{"tomcat": {}},
Matches: map[string][]string{"tomcat": {}},
Extracts: map[string][]string{},
}},
}}, Options: &protocols.ExecuterOptions{Progress: progressBar}},
@ -134,7 +135,7 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) {
firstInput = input
}, outputs: []*output.InternalWrappedEvent{
{OperatorsResult: &operators.Result{
Matches: map[string]struct{}{"tomcat": {}},
Matches: map[string][]string{"tomcat": {}},
Extracts: map[string][]string{},
}},
}}, Options: &protocols.ExecuterOptions{Progress: progressBar}},