New integration test + misc

.github/workflows/build.yaml

@@ -19,16 +19,7 @@ jobs:
           version: v1.33
           args: --timeout 5m
           working-directory: v2/
-
-          # Optional: working directory, useful for monorepos
-          # working-directory: somedir
-
-          # Optional: golangci-lint command line arguments.
-          # args: --issues-exit-code=0
-
-          # Optional: show only new issues if it's a pull request. The default value is `false`.
-          # only-new-issues: true
-
+        
   build:
     name: Build
     runs-on: ubuntu-latest
@@ -36,14 +27,17 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.14
+          go-version: 1.15
 
       - name: Check out code
         uses: actions/checkout@v2
 
       - name: Test
-        run: go test .
-        working-directory: v2/cmd/nuclei/
+        run: go test ./...
+        working-directory: v2/
+
+      - name: Integration Tests
+        run: bash integration_tests/run.sh
 
       - name: Build
         run: go build .

.github/workflows/release.yml

@@ -17,7 +17,7 @@ jobs:
         name: "Set up Go"
         uses: actions/setup-go@v2
         with: 
-          go-version: 1.14
+          go-version: 1.15
       - 
         env: 
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

v2/pkg/protocols/common/clusterer/clusterer_test.go

@@ -1,79 +0,0 @@
-package clusterer
-
-import (
-	"fmt"
-	"log"
-	"testing"
-
-	"github.com/logrusorgru/aurora"
-	"github.com/projectdiscovery/nuclei/v2/pkg/catalog"
-	"github.com/projectdiscovery/nuclei/v2/pkg/output"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit"
-	"github.com/projectdiscovery/nuclei/v2/pkg/templates"
-	"github.com/projectdiscovery/nuclei/v2/pkg/types"
-	"github.com/stretchr/testify/require"
-)
-
-func TestHTTPRequestsCluster(t *testing.T) {
-	catalogImpl := catalog.New("/Users/ice3man/nuclei-templates")
-	templatesList, err := catalogImpl.GetTemplatePath("/Users/ice3man/nuclei-templates")
-	require.Nil(t, err, "could not get templates")
-
-	_ = protocolinit.Init(&types.Options{})
-	list := make(map[string]*templates.Template)
-	for _, template := range templatesList {
-		executerOpts := protocols.ExecuterOptions{
-			Output:       &mockOutput{},
-			Options:      &types.Options{},
-			Progress:     nil,
-			Catalog:      catalogImpl,
-			RateLimiter:  nil,
-			IssuesClient: nil,
-			ProjectFile:  nil,
-		}
-		t, err := templates.Parse(template, executerOpts)
-		if err != nil {
-			continue
-		}
-		if _, ok := list[t.ID]; !ok {
-			list[t.ID] = t
-		} else {
-			log.Printf("Duplicate template found: %v\n", t)
-		}
-	}
-
-	totalClusterCount := 0
-	totalRequestsSentNew := 0
-	newRequests := Cluster(list)
-	for i, cluster := range newRequests {
-		if len(cluster) == 1 {
-			continue
-		}
-		fmt.Printf("[%d] cluster created:\n", i)
-		for _, request := range cluster {
-			totalClusterCount++
-			fmt.Printf("\t%v\n", request.ID)
-		}
-		totalRequestsSentNew++
-	}
-	fmt.Printf("Reduced %d requests to %d via clustering\n", totalClusterCount, totalRequestsSentNew)
-}
-
-type mockOutput struct{}
-
-// Close closes the output writer interface
-func (m *mockOutput) Close() {}
-
-// Colorizer returns the colorizer instance for writer
-func (m *mockOutput) Colorizer() aurora.Aurora {
-	return nil
-}
-
-// Write writes the event to file and/or screen.
-func (m *mockOutput) Write(*output.ResultEvent) error {
-	return nil
-}
-
-// Request writes a log the requests trace log
-func (m *mockOutput) Request(templateID, url, requestType string, err error) {}

v2/pkg/protocols/headless/operators_test.go

@@ -0,0 +1 @@
+package headless

v2/pkg/protocols/headless/request_test.go

@@ -1,55 +0,0 @@
-package headless
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/projectdiscovery/nuclei/v2/internal/testutils"
-	"github.com/projectdiscovery/nuclei/v2/pkg/operators"
-	"github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers"
-	"github.com/projectdiscovery/nuclei/v2/pkg/output"
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine"
-	"github.com/stretchr/testify/require"
-)
-
-func TestHeadlessExecuteWithResults(t *testing.T) {
-	options := testutils.DefaultOptions
-
-	testutils.Init(options)
-	templateID := "testing-headless"
-	request := &Request{
-		ID: templateID,
-		Steps: []*engine.Action{
-			{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}},
-			{ActionType: "waitload"},
-		},
-		Operators: operators.Operators{
-			Matchers: []*matchers.Matcher{{
-				Name:  "test",
-				Part:  "data",
-				Type:  "word",
-				Words: []string{"Example Domain"},
-			}},
-		},
-	}
-	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
-		ID:   templateID,
-		Info: map[string]interface{}{"severity": "low", "name": "test"},
-	})
-	options.Headless = true
-	browser, err := engine.New(options)
-	require.Nil(t, err, "could not create browser")
-	executerOpts.Browser = browser
-
-	err = request.Compile(executerOpts)
-	require.Nil(t, err, "could not compile headless request")
-
-	metadata := make(output.InternalEvent)
-	previous := make(output.InternalEvent)
-	err = request.ExecuteWithResults("https://example.com", metadata, previous, func(event *output.InternalWrappedEvent) {
-		for _, result := range event.Results {
-			fmt.Printf("Result: %+v\n", result)
-		}
-	})
-	require.Nil(t, err, "could not execute headless request")
-}

v2/pkg/protocols/network/operators.go

@@ -69,8 +69,8 @@ func (r *Request) responseToDSLMap(req, resp, raw, host, matched string) output.
 	data["host"] = host
 	data["matched"] = matched
 	data["request"] = req
-	data["data"] = resp
-	data["raw"] = raw
+	data["data"] = resp // Data is the last bytes read
+	data["raw"] = raw   // Raw is the full transaction data for network
 	data["template-id"] = r.options.TemplateID
 	data["template-info"] = r.options.TemplateInfo
 	return data

v2/pkg/protocols/network/operators_test.go

@@ -31,8 +31,8 @@ func TestResponseToDSLMap(t *testing.T) {
 
 	req := "test-data\r\n"
 	resp := "resp-data\r\n"
-	event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one", "test")
-	require.Len(t, event, 6, "could not get correct number of items in dsl map")
+	event := request.responseToDSLMap(req, resp, "test", "one.one.one.one", "one.one.one.one")
+	require.Len(t, event, 7, "could not get correct number of items in dsl map")
 	require.Equal(t, resp, event["data"], "could not get correct resp")
 }
 
@@ -73,7 +73,7 @@ func TestNetworkOperatorMatch(t *testing.T) {
 
 	t.Run("negative", func(t *testing.T) {
 		matcher := &matchers.Matcher{
-			Part:     "raw",
+			Part:     "data",
 			Type:     "word",
 			Negative: true,
 			Words:    []string{"random"},
@@ -87,7 +87,7 @@ func TestNetworkOperatorMatch(t *testing.T) {
 
 	t.Run("invalid", func(t *testing.T) {
 		matcher := &matchers.Matcher{
-			Part:  "raw",
+			Part:  "data",
 			Type:  "word",
 			Words: []string{"random"},
 		}
@@ -123,7 +123,7 @@ func TestNetworkOperatorExtract(t *testing.T) {
 
 	t.Run("extract", func(t *testing.T) {
 		extractor := &extractors.Extractor{
-			Part:  "raw",
+			Part:  "data",
 			Type:  "regex",
 			Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
 		}
@@ -162,12 +162,12 @@ func TestNetworkMakeResult(t *testing.T) {
 		Operators: operators.Operators{
 			Matchers: []*matchers.Matcher{{
 				Name:  "test",
-				Part:  "raw",
+				Part:  "data",
 				Type:  "word",
 				Words: []string{"STAT "},
 			}},
 			Extractors: []*extractors.Extractor{{
-				Part:  "raw",
+				Part:  "data",
 				Type:  "regex",
 				Regex: []string{"[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"},
 			}},

v2/pkg/protocols/network/request_test.go

@@ -25,12 +25,12 @@ func TestNetworkExecuteWithResults(t *testing.T) {
 		Operators: operators.Operators{
 			Matchers: []*matchers.Matcher{{
 				Name:  "test",
-				Part:  "raw",
+				Part:  "data",
 				Type:  "word",
 				Words: []string{"400 - Bad Request"},
 			}},
 			Extractors: []*extractors.Extractor{{
-				Part:  "raw",
+				Part:  "data",
 				Type:  "regex",
 				Regex: []string{"<h1>.*</h1>"},
 			}},

v2/pkg/protocols/offlinehttp/read_response.go

@@ -17,7 +17,7 @@ func readResponseFromString(data string) (*http.Response, error) {
 		if lastIndex == -1 {
 			return nil, errors.New("malformed raw http response")
 		}
-		final = data // choose last http/ in case of it being later.
+		final = data[lastIndex:] // choose last http/ in case of it being later.
 	}
 	return http.ReadResponse(bufio.NewReader(strings.NewReader(final)), nil)
 }