diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5e6435e0..956da30d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -19,16 +19,7 @@ jobs: version: v1.33 args: --timeout 5m working-directory: v2/ - - # Optional: working directory, useful for monorepos - # working-directory: somedir - - # Optional: golangci-lint command line arguments. - # args: --issues-exit-code=0 - - # Optional: show only new issues if it's a pull request. The default value is `false`. - # only-new-issues: true - + build: name: Build runs-on: ubuntu-latest @@ -36,14 +27,17 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.14 + go-version: 1.15 - name: Check out code uses: actions/checkout@v2 - name: Test - run: go test . - working-directory: v2/cmd/nuclei/ + run: go test ./... + working-directory: v2/ + + - name: Integration Tests + run: bash integration_tests/run.sh - name: Build run: go build . diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 70cb60a9..14d4ad1e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,7 @@ jobs: name: "Set up Go" uses: actions/setup-go@v2 with: - go-version: 1.14 + go-version: 1.15 - env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/v2/pkg/protocols/common/clusterer/clusterer_test.go b/v2/pkg/protocols/common/clusterer/clusterer_test.go deleted file mode 100644 index 6f69ec7b..00000000 --- a/v2/pkg/protocols/common/clusterer/clusterer_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package clusterer - -import ( - "fmt" - "log" - "testing" - - "github.com/logrusorgru/aurora" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog" - "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" - "github.com/projectdiscovery/nuclei/v2/pkg/templates" - "github.com/projectdiscovery/nuclei/v2/pkg/types" - "github.com/stretchr/testify/require" -) - -func TestHTTPRequestsCluster(t *testing.T) { - catalogImpl := catalog.New("/Users/ice3man/nuclei-templates") - templatesList, err := catalogImpl.GetTemplatePath("/Users/ice3man/nuclei-templates") - require.Nil(t, err, "could not get templates") - - _ = protocolinit.Init(&types.Options{}) - list := make(map[string]*templates.Template) - for _, template := range templatesList { - executerOpts := protocols.ExecuterOptions{ - Output: &mockOutput{}, - Options: &types.Options{}, - Progress: nil, - Catalog: catalogImpl, - RateLimiter: nil, - IssuesClient: nil, - ProjectFile: nil, - } - t, err := templates.Parse(template, executerOpts) - if err != nil { - continue - } - if _, ok := list[t.ID]; !ok { - list[t.ID] = t - } else { - log.Printf("Duplicate template found: %v\n", t) - } - } - - totalClusterCount := 0 - totalRequestsSentNew := 0 - newRequests := Cluster(list) - for i, cluster := range newRequests { - if len(cluster) == 1 { - continue - } - fmt.Printf("[%d] cluster created:\n", i) - for _, request := range cluster { - totalClusterCount++ - fmt.Printf("\t%v\n", request.ID) - } - totalRequestsSentNew++ - } - fmt.Printf("Reduced %d requests to %d via clustering\n", totalClusterCount, totalRequestsSentNew) -} - -type mockOutput struct{} - -// Close closes the output writer interface -func (m *mockOutput) Close() {} - -// Colorizer returns the colorizer instance for writer -func (m *mockOutput) Colorizer() aurora.Aurora { - return nil -} - -// Write writes the event to file and/or screen. -func (m *mockOutput) Write(*output.ResultEvent) error { - return nil -} - -// Request writes a log the requests trace log -func (m *mockOutput) Request(templateID, url, requestType string, err error) {} diff --git a/v2/pkg/protocols/headless/operators_test.go b/v2/pkg/protocols/headless/operators_test.go new file mode 100644 index 00000000..dede326b --- /dev/null +++ b/v2/pkg/protocols/headless/operators_test.go @@ -0,0 +1 @@ +package headless diff --git a/v2/pkg/protocols/headless/request_test.go b/v2/pkg/protocols/headless/request_test.go deleted file mode 100644 index b75ee392..00000000 --- a/v2/pkg/protocols/headless/request_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package headless - -import ( - "fmt" - "testing" - - "github.com/projectdiscovery/nuclei/v2/internal/testutils" - "github.com/projectdiscovery/nuclei/v2/pkg/operators" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" - "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" - "github.com/stretchr/testify/require" -) - -func TestHeadlessExecuteWithResults(t *testing.T) { - options := testutils.DefaultOptions - - testutils.Init(options) - templateID := "testing-headless" - request := &Request{ - ID: templateID, - Steps: []*engine.Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - }, - Operators: operators.Operators{ - Matchers: []*matchers.Matcher{{ - Name: "test", - Part: "data", - Type: "word", - Words: []string{"Example Domain"}, - }}, - }, - } - executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ - ID: templateID, - Info: map[string]interface{}{"severity": "low", "name": "test"}, - }) - options.Headless = true - browser, err := engine.New(options) - require.Nil(t, err, "could not create browser") - executerOpts.Browser = browser - - err = request.Compile(executerOpts) - require.Nil(t, err, "could not compile headless request") - - metadata := make(output.InternalEvent) - previous := make(output.InternalEvent) - err = request.ExecuteWithResults("https://example.com", metadata, previous, func(event *output.InternalWrappedEvent) { - for _, result := range event.Results { - fmt.Printf("Result: %+v\n", result) - } - }) - require.Nil(t, err, "could not execute headless request") -} diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index 135d9c02..a0a95ede 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -69,8 +69,8 @@ func (r *Request) responseToDSLMap(req, resp, raw, host, matched string) output. data["host"] = host data["matched"] = matched data["request"] = req - data["data"] = resp - data["raw"] = raw + data["data"] = resp // Data is the last bytes read + data["raw"] = raw // Raw is the full transaction data for network data["template-id"] = r.options.TemplateID data["template-info"] = r.options.TemplateInfo return data diff --git a/v2/pkg/protocols/network/operators_test.go b/v2/pkg/protocols/network/operators_test.go index c9da39a7..e729c777 100644 --- a/v2/pkg/protocols/network/operators_test.go +++ b/v2/pkg/protocols/network/operators_test.go @@ -31,8 +31,8 @@ func TestResponseToDSLMap(t *testing.T) { req := "test-data\r\n" resp := "resp-data\r\n" - event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", "test") - require.Len(t, event, 6, "could not get correct number of items in dsl map") + event := request.responseToDSLMap(req, resp, "test", "one.one.one.one", "one.one.one.one") + require.Len(t, event, 7, "could not get correct number of items in dsl map") require.Equal(t, resp, event["data"], "could not get correct resp") } @@ -73,7 +73,7 @@ func TestNetworkOperatorMatch(t *testing.T) { t.Run("negative", func(t *testing.T) { matcher := &matchers.Matcher{ - Part: "raw", + Part: "data", Type: "word", Negative: true, Words: []string{"random"}, @@ -87,7 +87,7 @@ func TestNetworkOperatorMatch(t *testing.T) { t.Run("invalid", func(t *testing.T) { matcher := &matchers.Matcher{ - Part: "raw", + Part: "data", Type: "word", Words: []string{"random"}, } @@ -123,7 +123,7 @@ func TestNetworkOperatorExtract(t *testing.T) { t.Run("extract", func(t *testing.T) { extractor := &extractors.Extractor{ - Part: "raw", + Part: "data", Type: "regex", Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, } @@ -162,12 +162,12 @@ func TestNetworkMakeResult(t *testing.T) { Operators: operators.Operators{ Matchers: []*matchers.Matcher{{ Name: "test", - Part: "raw", + Part: "data", Type: "word", Words: []string{"STAT "}, }}, Extractors: []*extractors.Extractor{{ - Part: "raw", + Part: "data", Type: "regex", Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"}, }}, diff --git a/v2/pkg/protocols/network/request_test.go b/v2/pkg/protocols/network/request_test.go index fdb67f62..152ddbad 100644 --- a/v2/pkg/protocols/network/request_test.go +++ b/v2/pkg/protocols/network/request_test.go @@ -25,12 +25,12 @@ func TestNetworkExecuteWithResults(t *testing.T) { Operators: operators.Operators{ Matchers: []*matchers.Matcher{{ Name: "test", - Part: "raw", + Part: "data", Type: "word", Words: []string{"400 - Bad Request"}, }}, Extractors: []*extractors.Extractor{{ - Part: "raw", + Part: "data", Type: "regex", Regex: []string{"