fix missing port in matched ssl templates (#3380)

* add openssl support + fix missing port * fix failing tests * go mod update * workflow update --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
2023-03-04 04:47:41 +05:30 · 2023-03-04 04:47:41 +05:30 · 3e53087617
parent ecf3671982
commit 3e53087617
14 changed files with 62 additions and 93 deletions
--- a/.github/workflows/build-test.yml
+++ b/.github/workflows/build-test.yml
@ -13,7 +13,7 @@ jobs:
    strategy:
      matrix:
        go-version: [1.19.x]
-        os: [ubuntu-latest, windows-latest, macOS-12]
+        os: [ubuntu-latest-16-cores, windows-latest-8-cores, macOS-12]

    runs-on: ${{ matrix.os }}
    steps:
--- a/.github/workflows/changelog-indexer.yaml
+++ b/.github/workflows/changelog-indexer.yaml
@ -8,7 +8,7 @@ on:

 jobs:
  index:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-16-cores
    steps:
      - uses: actions/setup-go@v3
        with:
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@ -7,7 +7,7 @@ on:
 jobs:
  analyze:
    name: Analyze
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-16-cores
    permissions:
      actions: read
      contents: read
--- a/.github/workflows/dockerhub-push.yml
+++ b/.github/workflows/dockerhub-push.yml
@ -9,7 +9,7 @@ on:

 jobs:
  docker:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-16-cores
    steps:
      - name: Git Checkout
        uses: actions/checkout@v3
--- a/.github/workflows/functional-test.yml
+++ b/.github/workflows/functional-test.yml
@ -13,7 +13,7 @@ jobs:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
-        os: [ubuntu-latest, windows-latest, macOS-latest]
+        os: [ubuntu-latest-16-cores, windows-latest-8-cores, macOS-12]
    steps:
      - name: Set up Go
        uses: actions/setup-go@v3
--- a/.github/workflows/lint-test.yml
+++ b/.github/workflows/lint-test.yml
@ -7,7 +7,7 @@ on:
 jobs:
  lint:
    name: Lint Test
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-16-cores
    steps:
      - name: Set up Go
        uses: actions/setup-go@v3
--- a/.github/workflows/publish-docs.yaml
+++ b/.github/workflows/publish-docs.yaml
@ -8,7 +8,7 @@ on:

 jobs:
  docs:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-16-cores
    steps:
      - name: Checkout code
        uses: actions/checkout@v3
--- a/.github/workflows/release-binary.yml
+++ b/.github/workflows/release-binary.yml
@ -8,7 +8,7 @@ on:

 jobs: 
  release: 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-16-cores

    steps: 
      - uses: actions/checkout@v3
--- a/.github/workflows/sonarcloud.yml
+++ b/.github/workflows/sonarcloud.yml
@ -1,38 +0,0 @@
-name: 👮🏼‍♂️ Sonarcloud
-
-on:
-  pull_request:
-    paths:
-      - '**.go'
-  workflow_dispatch:
-
-jobs:
-  sonarcloud:
-    name: SonarCloud
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
-     
-      - name: "Set up Go"
-        uses: actions/setup-go@v3
-        with: 
-          go-version: 1.19
-      
-      - name: Run unit Tests
-        working-directory: v2/
-        run: |
-          go test -coverprofile=cov.out ./...
-
-      - name: Run Gosec Security Scanner
-        working-directory: v2/
-        run: |
-          go install github.com/securego/gosec/cmd/gosec@latest
-          gosec -no-fail -fmt=sonarqube -out report.json ./...
-     
-      - name: SonarCloud Scan
-        uses: SonarSource/sonarcloud-github-action@master
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
--- a/.github/workflows/template-validate.yml
+++ b/.github/workflows/template-validate.yml
@ -8,7 +8,7 @@ on:

 jobs:
  build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-16-cores
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-go@v3
--- a/v2/go.mod
+++ b/v2/go.mod
@ -25,7 +25,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/clistats v0.0.12
 	github.com/projectdiscovery/fastdialer v0.0.22
-	github.com/projectdiscovery/hmap v0.0.7
+	github.com/projectdiscovery/hmap v0.0.8
 	github.com/projectdiscovery/interactsh v1.0.6-0.20220827132222-460cc6270053
 	github.com/projectdiscovery/rawhttp v0.1.9
 	github.com/projectdiscovery/retryabledns v1.0.21
@ -71,7 +71,7 @@ require (
 	github.com/mholt/archiver v3.1.1+incompatible
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/projectdiscovery/fasttemplate v0.0.2
-	github.com/projectdiscovery/goflags v0.1.6
+	github.com/projectdiscovery/goflags v0.1.7
 	github.com/projectdiscovery/gologger v1.1.8
 	github.com/projectdiscovery/httpx v1.2.7
 	github.com/projectdiscovery/mapcidr v1.1.0
@ -108,7 +108,7 @@ require (
 	github.com/karlseguin/expect v1.0.8 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/pjbgf/sha1cd v0.2.3 // indirect
-	github.com/projectdiscovery/asnmap v1.0.0 // indirect
+	github.com/projectdiscovery/asnmap v1.0.1 // indirect
 	github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 // indirect
 	github.com/projectdiscovery/freeport v0.0.4 // indirect
 	github.com/prometheus/client_golang v1.14.0 // indirect
--- a/v2/go.sum
+++ b/v2/go.sum
@ -529,8 +529,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
-github.com/projectdiscovery/asnmap v1.0.0 h1:h9aUEHT3gEWgeTxDCd0UMxBw1yPthDwL1ogqUWnkBXo=
-github.com/projectdiscovery/asnmap v1.0.0/go.mod h1:m+qedSAZERz7Ds942hWANjU9kg4ADAikFHfwSXN1Ey8=
+github.com/projectdiscovery/asnmap v1.0.1 h1:mRFrFulfI5ZjYx0uyWcWKyCPRExDXInmBKpZlwxwRgE=
+github.com/projectdiscovery/asnmap v1.0.1/go.mod h1:qEowdtLF9JQqQ4/tHTt1HGAY0RnanLfU9DcAvrLic0o=
 github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4 h1:EsrQ/zkotVodSJLOch3pV/UYt1vQcwyIs5HX0sm1ljE=
 github.com/projectdiscovery/blackrock v0.0.0-20221025011524-9e4efe804fb4/go.mod h1:5tNGQP9kOfW+X5+40pZP8aqPYLHs45nJkFaSHLxdeH8=
 github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 h1:QtTPPx0uu42AsQJiXT86/wqdHS7/iVcgz1VM38tjv20=
@ -545,12 +545,12 @@ github.com/projectdiscovery/fileutil v0.0.3 h1:GSsoey4p8ZHIRxWF2VXh4mhLr+wfEkpJw
 github.com/projectdiscovery/fileutil v0.0.3/go.mod h1:GLejWd3YerG3RNYD/Hk2pJlytlYRgHdkWfWUAdCH2YQ=
 github.com/projectdiscovery/freeport v0.0.4 h1:H4VrK/7hUcC1zbg46zv9iSMBACBDpUqcHkV+FUyXISw=
 github.com/projectdiscovery/freeport v0.0.4/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE=
-github.com/projectdiscovery/goflags v0.1.6 h1:EXigzX4lJmn/fLMnULdc03O7WW+DjiYZhNgdGvfg+Z4=
-github.com/projectdiscovery/goflags v0.1.6/go.mod h1:yILgA7gbrHuTpIvMfikbivzoxkyxBD1Y5/PRHiGTIFk=
+github.com/projectdiscovery/goflags v0.1.7 h1:KvMs1KnXMWRhJiJj6K+cWurTWpbDYYlo+1cyGvf1YCc=
+github.com/projectdiscovery/goflags v0.1.7/go.mod h1:yILgA7gbrHuTpIvMfikbivzoxkyxBD1Y5/PRHiGTIFk=
 github.com/projectdiscovery/gologger v1.1.8 h1:CFlCzGlqAhPqWIrAXBt1OVh5jkMs1qgoR/z4xhdzLNE=
 github.com/projectdiscovery/gologger v1.1.8/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw=
-github.com/projectdiscovery/hmap v0.0.7 h1:pGNR+XLEOEDM3He3P0PXzszA0vCxNocXHLGeuA4Gf28=
-github.com/projectdiscovery/hmap v0.0.7/go.mod h1:xXSxFWk/zyiBrreQqgLTEnGYhufafVcCSXESBnqKOBU=
+github.com/projectdiscovery/hmap v0.0.8 h1:4Jn3H0gHeibJAvJuyVIZMszzALOurd8PKmgRrXfuhLc=
+github.com/projectdiscovery/hmap v0.0.8/go.mod h1:6V6eZru59wNZVtJB/NeiGUaZxBIR8vD97Ok2o9mXsyk=
 github.com/projectdiscovery/httpx v1.2.7 h1:rDB+uKpWE/e6nuLDM341ZNzaaMgwcEcc+A9Q5R6RL3s=
 github.com/projectdiscovery/httpx v1.2.7/go.mod h1:QW8mKw6PzMBb62T5fT9w75hhVBcjoyyVTDJhefZcX50=
 github.com/projectdiscovery/interactsh v1.0.6-0.20220827132222-460cc6270053 h1:8Dr2q8BWxNgG5EO/YTyNyL83xzmYSDwysx4pMZzZx7I=
--- a/v2/pkg/input/input.go
+++ b/v2/pkg/input/input.go
@ -2,7 +2,6 @@ package input

 import (
 	"net"
-	"net/url"
 	"path/filepath"
 	"strings"

@ -11,6 +10,7 @@ import (
 	fileutil "github.com/projectdiscovery/utils/file"
 	"github.com/projectdiscovery/utils/ports"
 	stringsutil "github.com/projectdiscovery/utils/strings"
+	urlutil "github.com/projectdiscovery/utils/url"
 )

 // Helper is a structure for helping with input transformation
@ -45,8 +45,6 @@ func (h *Helper) Transform(input string, protocol templateTypes.ProtocolType) st
 		return h.convertInputToType(input, typeURL, "")
 	case templateTypes.NetworkProtocol:
 		return h.convertInputToType(input, typeHostWithOptionalPort, "")
-	case templateTypes.SSLProtocol:
-		return h.convertInputToType(input, typeHostWithPort, "443")
 	case templateTypes.WebsocketProtocol:
 		return h.convertInputToType(input, typeWebsocket, "")
 	}
@ -68,7 +66,7 @@ const (
 // Various formats are supported for inputs and their transformation
 func (h *Helper) convertInputToType(input string, inputType inputType, defaultPort string) string {
 	isURL := strings.Contains(input, "://")
-	uri, _ := url.Parse(input)
+	uri, _ := urlutil.Parse(input)

 	var host, port string
 	if isURL && uri != nil {
--- a/v2/pkg/protocols/ssl/ssl.go
+++ b/v2/pkg/protocols/ssl/ssl.go
@ -3,8 +3,6 @@ package ssl
 import (
 	"fmt"
 	"net"
-	"net/url"
-	"strings"
 	"time"

 	"github.com/fatih/structs"
@ -32,6 +30,10 @@ import (
 	"github.com/projectdiscovery/nuclei/v2/pkg/types"
 	"github.com/projectdiscovery/tlsx/pkg/tlsx"
 	"github.com/projectdiscovery/tlsx/pkg/tlsx/clients"
+	"github.com/projectdiscovery/tlsx/pkg/tlsx/openssl"
+	errorutil "github.com/projectdiscovery/utils/errors"
+	stringsutil "github.com/projectdiscovery/utils/strings"
+	urlutil "github.com/projectdiscovery/utils/url"
 )

 // Request is a request for the SSL protocol
@ -70,6 +72,7 @@ type Request struct {
 	//   - "ctls"
 	//   - "ztls"
 	//   - "auto"
+	//	 - "openssl" # reverts to "auto" is openssl is not installed
 	ScanMode string `yaml:"scan_mode,omitempty" json:"scan_mode,omitempty" jsonschema:"title=Scan Mode,description=Scan Mode - auto if not specified.,enum=ctls,enum=ztls,enum=auto"`

 	// cache any variables that may be needed for operation.
@ -95,15 +98,32 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {

 	client, err := networkclientpool.Get(options.Options, &networkclientpool.Configuration{})
 	if err != nil {
-		return errors.Wrap(err, "could not get network client")
+		return errorutil.NewWithTag("ssl", "could not get network client").Wrap(err)
 	}
 	request.dialer = client
+	switch {
+	//validate scanmode
+	case request.ScanMode == "":
+		request.ScanMode = "auto"
+
+	case !stringsutil.EqualFoldAny(request.ScanMode, "auto", "openssl", "ztls", "ctls"):
+		return errorutil.NewWithTag(request.TemplateID, "template %v does not contain valid scan-mode", request.TemplateID)
+
+	case request.ScanMode == "openssl" && !openssl.IsAvailable():
+		// if openssl is not installed instead of failing "auto" scanmode is used
+		request.ScanMode = "auto"
+
+	case options.Options.ZTLS && request.ScanMode == "ctls":
+		// only override if scanmode in template is "ctls" since auto internally uses ztls as fallback
+		request.ScanMode = "ztls"
+	}

 	tlsxOptions := &clients.Options{
 		AllCiphers:        true,
-		ScanMode:          "auto",
+		ScanMode:          request.ScanMode,
 		Expired:           true,
 		SelfSigned:        true,
+		Revoked:           true,
 		MisMatched:        true,
 		MinVersion:        request.MinVersion,
 		MaxVersion:        request.MaxVersion,
@ -115,14 +135,10 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 		ClientHello:       true,
 		ServerHello:       true,
 	}
-	if options.Options.ZTLS {
-		tlsxOptions.ScanMode = "ztls"
-	} else if request.ScanMode != "" {
-		tlsxOptions.ScanMode = request.ScanMode
-	}
+
 	tlsxService, err := tlsx.New(tlsxOptions)
 	if err != nil {
-		return errors.Wrap(err, "could not create tlsx service")
+		return errorutil.NewWithTag(request.TemplateID, "could not create tlsx service")
 	}
 	request.tlsx = tlsxService

@ -131,7 +147,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 		compiled.ExcludeMatchers = options.ExcludeMatchers
 		compiled.TemplateID = options.TemplateID
 		if err := compiled.Compile(); err != nil {
-			return errors.Wrap(err, "could not compile operators")
+			return errorutil.NewWithTag(request.TemplateID, "could not compile operators got %v", err)
 		}
 		request.CompiledOperators = compiled
 	}
@ -157,7 +173,7 @@ func (request *Request) GetID() string {
 func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
 	hostPort, err := getAddress(input.MetaInput.Input)
 	if err != nil {
-		return nil
+		return err
 	}
 	hostname, port, _ := net.SplitHostPort(hostPort)

@ -189,7 +205,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 	addressToDial := string(finalAddress)
 	host, port, err := net.SplitHostPort(addressToDial)
 	if err != nil {
-		return errors.Wrap(err, "could not split input host port")
+		return errorutil.NewWithErr(err).Msgf("could not split input host port")
 	}

 	var hostIp string
@ -203,7 +219,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 	if err != nil {
 		requestOptions.Output.Request(requestOptions.TemplateID, input.MetaInput.Input, request.Type().String(), err)
 		requestOptions.Progress.IncrementFailedRequestsBy(1)
-		return errors.Wrap(err, "could not connect to server")
+		return errorutil.NewWithTag(request.TemplateID, "could not connect to server").Wrap(err)
 	}

 	requestOptions.Output.Request(requestOptions.TemplateID, hostPort, request.Type().String(), err)
@ -223,10 +239,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 	jsonDataString := string(jsonData)

 	data := make(map[string]interface{})
-
+	for k, v := range payloadValues {
+		data[k] = v
+	}
 	data["type"] = request.Type().String()
 	data["response"] = jsonDataString
-	data["host"] = input
+	data["host"] = input.MetaInput.Input
 	data["matched"] = addressToDial
 	if input.MetaInput.CustomIP != "" {
 		data["ip"] = hostIp
@ -236,9 +254,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 	data["template-path"] = requestOptions.TemplatePath
 	data["template-id"] = requestOptions.TemplateID
 	data["template-info"] = requestOptions.TemplateInfo
-	for k, v := range payloadValues {
-		data[k] = v
-	}

 	// Convert response to key value pairs and first cert chain item as well
 	responseParsed := structs.New(response)
@ -286,21 +301,15 @@ var RequestPartDefinitions = map[string]string{

 // getAddress returns the address of the host to make request to
 func getAddress(toTest string) (string, error) {
-	if strings.Contains(toTest, "://") {
-		parsed, err := url.Parse(toTest)
-		if err != nil {
-			return "", err
-		}
-		_, port, _ := net.SplitHostPort(parsed.Host)
-
-		if strings.ToLower(parsed.Scheme) == "https" && port == "" {
-			toTest = net.JoinHostPort(parsed.Host, "443")
-		} else {
-			toTest = parsed.Host
-		}
+	urlx, err := urlutil.Parse(toTest)
+	if err != nil {
+		// use given input instead of url parsing failure
 		return toTest, nil
 	}
-	return toTest, nil
+	if urlx.Port() == "" {
+		urlx.UpdatePort("443")
+	}
+	return urlx.Host, nil
 }

 // Match performs matching operation for a matcher on model and returns:
@ -337,7 +346,7 @@ func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent
 		Info:             wrapped.InternalEvent["template-info"].(model.Info),
 		Type:             types.ToString(wrapped.InternalEvent["type"]),
 		Host:             types.ToString(wrapped.InternalEvent["host"]),
-		Matched:          types.ToString(wrapped.InternalEvent["host"]),
+		Matched:          types.ToString(wrapped.InternalEvent["matched"]),
 		Metadata:         wrapped.OperatorsResult.PayloadValues,
 		ExtractedResults: wrapped.OperatorsResult.OutputExtracts,
 		Timestamp:        time.Now(),