diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..4d5617f --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,34 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: "chore" + include: "scope" + + # Maintain dependencies for go modules + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: "chore" + include: "scope" + + # Maintain dependencies for docker + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: "chore" + include: "scope" diff --git a/.golangci.yml b/.golangci.yml index 62ead99..d5e9089 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -108,9 +108,15 @@ linters: # - testpackage # - wsl +issues: + exclude-use-default: false + exclude: + # should have a package comment, unless it's in another file for this package (golint) + - 'in another file for this package' + # golangci.com configuration # https://github.com/golangci/golangci/wiki/Configuration service: - golangci-lint-version: 1.33.x # use the fixed version to not introduce new linters unexpectedly + golangci-lint-version: 1.31.x # use the fixed version to not introduce new linters unexpectedly prepare: - echo "here I can run custom commands, but no preparation needed for this repo" diff --git a/DISCLAIMER.md b/DISCLAIMER.md index 2280435..cb8575d 100644 --- a/DISCLAIMER.md +++ b/DISCLAIMER.md @@ -2,7 +2,7 @@ Subfinder leverages multiple open APIs, it is developed for individuals to help them for research or internal work. If you wish to incorporate this tool into a commercial offering or purposes, you must agree to the Terms of the leveraged services: -- Project Sonar / Bufferover: https://opendata.rapid7.com/about +- Project Sonar / Bufferover: https://opendata.rapid7.com/about / [https://tls.bufferover.run](https://tls.bufferover.run/dns?q) - CommonCrawl: https://commoncrawl.org/terms-of-use/full - certspotter: https://sslmate.com/terms - dnsdumpster: https://hackertarget.com/terms diff --git a/Dockerfile b/Dockerfile index 15eba07..cfeec0c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,8 @@ FROM golang:1.13.4-alpine3.10 AS build-env MAINTAINER Ice3man (nizamul@projectdiscovery.io) RUN apk add --no-cache --upgrade git openssh-client ca-certificates -RUN go get -u github.com/golang/dep/cmd/dep -RUN go get -u github.com/projectdiscovery/subfinder/v2/cmd/subfinder +RUN go get -v github.com/golang/dep/cmd/dep +RUN GO111MODULE=on go get -v github.com/projectdiscovery/v2/cmd/subfinder FROM alpine:latest COPY --from=build-env /go/bin/subfinder /usr/local/bin/ diff --git a/LICENSE.md b/LICENSE.md index 5e85035..b22968b 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ MIT License -Copyright (c) Exposed Atoms Pvt Ltd +Copyright (c) 2021 ProjectDiscovery, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index cae423f..c05dd2b 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ The installation is easy. You can download the pre-built binaries for different subfinder requires **go1.14+** to install successfully. Run the following command to get the repo - ```sh -GO111MODULE=on go get -u -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder +GO111MODULE=on go get -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder ``` ### From Github @@ -288,6 +288,10 @@ For example, this runs the tool against uber.com and output the results to your docker run -v $HOME/.config/subfinder:/root/.config/subfinder -it projectdiscovery/subfinder -d uber.com > uber.com.txt ``` +### Resources + +- [Recon with Me !!!](https://dhiyaneshgeek.github.io/bug/bounty/2020/02/06/recon-with-me/) + # License subfinder is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[Thanks.md](https://github.com/projectdiscovery/subfinder/blob/master/THANKS.md)** file for more details. diff --git a/v2/cmd/subfinder/main.go b/v2/cmd/subfinder/main.go index d955d51..ac42492 100644 --- a/v2/cmd/subfinder/main.go +++ b/v2/cmd/subfinder/main.go @@ -15,11 +15,11 @@ func main() { newRunner, err := runner.NewRunner(options) if err != nil { - gologger.Fatalf("Could not create runner: %s\n", err) + gologger.Fatal().Msgf("Could not create runner: %s\n", err) } err = newRunner.RunEnumeration(context.Background()) if err != nil { - gologger.Fatalf("Could not run enumeration: %s\n", err) + gologger.Fatal().Msgf("Could not run enumeration: %s\n", err) } } diff --git a/v2/go.mod b/v2/go.mod index 688c481..9ecd67f 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -13,8 +13,13 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/chaos-client v0.1.6 github.com/projectdiscovery/dnsx v1.0.1 + github.com/miekg/dns v1.1.38 + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/pkg/errors v0.9.1 + github.com/projectdiscovery/chaos-client v0.1.7 github.com/projectdiscovery/fdmax v0.0.3 - github.com/projectdiscovery/gologger v1.0.1 + github.com/projectdiscovery/gologger v1.1.3 github.com/rs/xid v1.2.1 github.com/stretchr/testify v1.6.1 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 @@ -24,4 +29,12 @@ require ( golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 + github.com/stretchr/testify v1.7.0 + github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 + golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect + golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect + golang.org/x/sync v0.0.0-20201207232520-09787c993a3a // indirect + golang.org/x/sys v0.0.0-20210217105451-b926d437f341 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/v2/go.sum b/v2/go.sum index e0205fb..13e23bb 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -27,6 +27,8 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.35 h1:oTfOaDH+mZkdcgdIjH6yBajRGtIwcwcaR+rt23ZSrJs= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.38 h1:MtIY+fmHUVVgv1AXzmKMWcwdCYxTRPG1EDjpqF4RCEw= +github.com/miekg/dns v1.1.38/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -55,6 +57,13 @@ github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLY github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= github.com/projectdiscovery/retryabledns v1.0.5 h1:bQivGy5CuqKlwcxRkgA5ENincqIed/BR2sA6t2gdwuI= github.com/projectdiscovery/retryabledns v1.0.5/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ= +github.com/projectdiscovery/chaos-client v0.1.7 h1:8M1LazT+lD3qJP1ERWXiVzQn9lPMO3p6v3qsiGNKUlQ= +github.com/projectdiscovery/chaos-client v0.1.7/go.mod h1:F5omaoJh/vMvWnZhKD4zFFA5ti+RPwUletwepKSyfxk= +github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= +github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= +github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= +github.com/projectdiscovery/gologger v1.1.3 h1:rKWZW2QUigRV1jnlWwWJbJRvz8b+T/+bB5qemDGGBJU= +github.com/projectdiscovery/gologger v1.1.3/go.mod h1:jdXflz3TLB8bcVNzb0v26TztI9KPz8Lr4BVdUhNUs6E= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -64,6 +73,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -75,6 +86,8 @@ golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9 h1:umElSU9WZirRdgu2yFHY0a golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9 h1:sYNJzB4J8toYPQTM6pAkcmBRgw9SnQKP9oXCHfgy604= golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -84,6 +97,8 @@ golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201216054612-986b41b23924 h1:QsnDpLLOKwHBBDa8nDws4DYNc/ryVW2vCpxCs09d4PY= golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= @@ -99,6 +114,11 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3 h1:kzM6+9dur93BcC2kVlYl34cHU golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US3H2JFBj920Ot3rtDjs= golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3 h1:kzM6+9dur93BcC2kVlYl34cHU+TYZLanmpSJHVMmL64= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210217105451-b926d437f341 h1:2/QtM1mL37YmcsT8HaDNHDgTqqFVw+zr8UzMiBVLzYU= +golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -118,3 +138,8 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index d4be059..4843f33 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -47,7 +47,7 @@ func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, timeo wg.Wait() for source, data := range timeTaken { - gologger.Verbosef(data, source) + gologger.Verbose().Msgf(data, source) } close(results) diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 2b1d1c3..30a67e3 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -7,10 +7,8 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/archiveis" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/binaryedge" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bufferover" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/cebaidu" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/censys" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotter" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotterold" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chaos" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh" @@ -45,9 +43,7 @@ var DefaultSources = []string{ "alienvault", "anubis", "bufferover", - "cebaidu", "certspotter", - "certspotterold", "censys", "chaos", "crtsh", @@ -72,9 +68,7 @@ var DefaultRecursiveSources = []string{ "alienvault", "binaryedge", "bufferover", - "cebaidu", "certspotter", - "certspotterold", "crtsh", "dnsdumpster", "hackertarget", @@ -94,10 +88,8 @@ var DefaultAllSources = []string{ "archiveis", "binaryedge", "bufferover", - "cebaidu", "censys", "certspotter", - "certspotterold", "chaos", "commoncrawl", "crtsh", @@ -159,14 +151,10 @@ func (a *Agent) addSources(sources []string) { a.sources[source] = &binaryedge.Source{} case "bufferover": a.sources[source] = &bufferover.Source{} - case "cebaidu": - a.sources[source] = &cebaidu.Source{} case "censys": a.sources[source] = &censys.Source{} case "certspotter": a.sources[source] = &certspotter.Source{} - case "certspotterold": - a.sources[source] = &certspotterold.Source{} case "chaos": a.sources[source] = &chaos.Source{} case "commoncrawl": diff --git a/v2/pkg/resolve/resolve.go b/v2/pkg/resolve/resolve.go index 2720533..76fa43b 100644 --- a/v2/pkg/resolve/resolve.go +++ b/v2/pkg/resolve/resolve.go @@ -104,14 +104,18 @@ func (r *ResolutionPool) resolveWorker() { continue } + var skip bool for _, host := range hosts { // Ignore the host if it exists in wildcard ips map if _, ok := r.wildcardIPs[host]; ok { //nolint:staticcheck //search alternatives for "comma ok" - continue + skip = true + break } } - r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source} + if !skip { + r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source} + } } r.wg.Done() } diff --git a/v2/pkg/runner/banners.go b/v2/pkg/runner/banners.go index b7ff55e..e1dc19b 100644 --- a/v2/pkg/runner/banners.go +++ b/v2/pkg/runner/banners.go @@ -10,27 +10,27 @@ const banner = ` _ __ _ _ ____ _| |__ / _(_)_ _ __| |___ _ _ (_-< || | '_ \ _| | ' \/ _ / -_) '_| -/__/\_,_|_.__/_| |_|_||_\__,_\___|_| v2.4.5 +/__/\_,_|_.__/_| |_|_||_\__,_\___|_| v2.4.6 ` // Version is the current version of subfinder -const Version = `2.4.5` +const Version = `2.4.6` // showBanner is used to show the banner to the user func showBanner() { - gologger.Printf("%s\n", banner) - gologger.Printf("\t\tprojectdiscovery.io\n\n") + gologger.Print().Msgf("%s\n", banner) + gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n") - gologger.Labelf("Use with caution. You are responsible for your actions\n") - gologger.Labelf("Developers assume no liability and are not responsible for any misuse or damage.\n") - gologger.Labelf("By using subfinder, you also agree to the terms of the APIs used.\n\n") + gologger.Print().Msgf("Use with caution. You are responsible for your actions\n") + gologger.Print().Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n") + gologger.Print().Msgf("By using subfinder, you also agree to the terms of the APIs used.\n\n") } // normalRunTasks runs the normal startup tasks func (options *Options) normalRunTasks() { configFile, err := UnmarshalRead(options.ConfigFile) if err != nil { - gologger.Fatalf("Could not read configuration file %s: %s\n", options.ConfigFile, err) + gologger.Fatal().Msgf("Could not read configuration file %s: %s\n", options.ConfigFile, err) } // If we have a different version of subfinder installed @@ -43,7 +43,7 @@ func (options *Options) normalRunTasks() { err = configFile.MarshalWrite(options.ConfigFile) if err != nil { - gologger.Fatalf("Could not update configuration file to %s: %s\n", options.ConfigFile, err) + gologger.Fatal().Msgf("Could not update configuration file to %s: %s\n", options.ConfigFile, err) } } options.YAMLConfig = configFile @@ -67,9 +67,9 @@ func (options *Options) firstRunTasks() { err := config.MarshalWrite(options.ConfigFile) if err != nil { - gologger.Fatalf("Could not write configuration file to %s: %s\n", options.ConfigFile, err) + gologger.Fatal().Msgf("Could not write configuration file to %s: %s\n", options.ConfigFile, err) } options.YAMLConfig = config - gologger.Infof("Configuration file saved to %s\n", options.ConfigFile) + gologger.Info().Msgf("Configuration file saved to %s\n", options.ConfigFile) } diff --git a/v2/pkg/runner/chaosuploader.go b/v2/pkg/runner/chaosuploader.go index eab1e17..442bd8f 100644 --- a/v2/pkg/runner/chaosuploader.go +++ b/v2/pkg/runner/chaosuploader.go @@ -42,7 +42,7 @@ func (r *Runner) UploadToChaos(ctx context.Context, reader io.Reader) error { defer func() { _, err := io.Copy(ioutil.Discard, resp.Body) if err != nil { - gologger.Warningf("Could not discard response body: %s\n", err) + gologger.Warning().Msgf("Could not discard response body: %s\n", err) return } resp.Body.Close() diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go index 3183ecf..34402f0 100644 --- a/v2/pkg/runner/enumerate.go +++ b/v2/pkg/runner/enumerate.go @@ -18,7 +18,7 @@ const maxNumCount = 2 // EnumerateSingleDomain performs subdomain enumeration against a single domain func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output string, appendToFile bool) error { - gologger.Infof("Enumerating subdomains for %s\n", domain) + gologger.Info().Msgf("Enumerating subdomains for %s\n", domain) // Get the API keys for sources from the configuration // and also create the active resolving engine for the domain. @@ -32,7 +32,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin err := resolutionPool.InitWildcards(domain) if err != nil { // Log the error but don't quit. - gologger.Warningf("Could not get wildcards for domain %s: %s\n", domain, err) + gologger.Warning().Msgf("Could not get wildcards for domain %s: %s\n", domain, err) } } @@ -51,7 +51,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin for result := range passiveResults { switch result.Type { case subscraping.Error: - gologger.Warningf("Could not run source %s: %s\n", result.Source, result.Error) + gologger.Warning().Msgf("Could not run source %s: %s\n", result.Source, result.Error) case subscraping.Subdomain: // Validate the subdomain found and remove wildcards from if !strings.HasSuffix(result.Value, "."+domain) { @@ -65,7 +65,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin // Log the verbose message about the found subdomain per source if _, ok := sourceMap[subdomain][result.Source]; !ok { - gologger.Verbosef("%s\n", result.Source, subdomain) + gologger.Verbose().Msgf("%s\n", result.Source, subdomain) } sourceMap[subdomain][result.Source] = struct{}{} @@ -103,7 +103,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin for result := range resolutionPool.Results { switch result.Type { case resolve.Error: - gologger.Warningf("Could not resolve host: %s\n", result.Error) + gologger.Warning().Msgf("Could not resolve host: %s\n", result.Error) case resolve.Subdomain: // Add the found subdomain to a map. if _, ok := foundResults[result.Host]; !ok { @@ -133,16 +133,16 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin } } if err != nil { - gologger.Errorf("Could not verbose results for %s: %s\n", domain, err) + gologger.Error().Msgf("Could not verbose results for %s: %s\n", domain, err) return err } // Show found subdomain count in any case. duration := durafmt.Parse(time.Since(now)).LimitFirstN(maxNumCount).String() if r.options.RemoveWildcard { - gologger.Infof("Found %d subdomains for %s in %s\n", len(foundResults), domain, duration) + gologger.Info().Msgf("Found %d subdomains for %s in %s\n", len(foundResults), domain, duration) } else { - gologger.Infof("Found %d subdomains for %s in %s\n", len(uniqueMap), domain, duration) + gologger.Info().Msgf("Found %d subdomains for %s in %s\n", len(uniqueMap), domain, duration) } // In case the user has specified to upload to chaos, write everything to a temporary buffer and upload @@ -151,14 +151,14 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin err := outputter.WriteForChaos(uniqueMap, buf) // If an error occurs, do not interrupt, continue to check if user specified an output file if err != nil { - gologger.Errorf("Could not prepare results for chaos %s\n", err) + gologger.Error().Msgf("Could not prepare results for chaos %s\n", err) } else { // no error in writing host output, upload to chaos err = r.UploadToChaos(ctx, buf) if err != nil { - gologger.Errorf("Could not upload results to chaos %s\n", err) + gologger.Error().Msgf("Could not upload results to chaos %s\n", err) } else { - gologger.Infof("Input processed successfully and subdomains with valid records will be updated to chaos dataset.\n") + gologger.Info().Msgf("Input processed successfully and subdomains with valid records will be updated to chaos dataset.\n") } // clear buffer buf.Reset() @@ -168,7 +168,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin if output != "" { file, err := outputter.createFile(output, appendToFile) if err != nil { - gologger.Errorf("Could not create file %s for %s: %s\n", output, domain, err) + gologger.Error().Msgf("Could not create file %s for %s: %s\n", output, domain, err) return err } @@ -188,7 +188,7 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output strin } } if err != nil { - gologger.Errorf("Could not write results to file %s for %s: %s\n", output, domain, err) + gologger.Error().Msgf("Could not write results to file %s for %s: %s\n", output, domain, err) return err } } diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index ee76d40..321befc 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -49,7 +49,7 @@ func ParseOptions() *Options { config, err := GetConfigDirectory() if err != nil { // This should never be reached - gologger.Fatalf("Could not get user home: %s\n", err) + gologger.Fatal().Msgf("Could not get user home: %s\n", err) } flag.BoolVar(&options.Verbose, "v", false, "Show Verbose output") @@ -89,7 +89,7 @@ func ParseOptions() *Options { showBanner() if options.Version { - gologger.Infof("Current Version: %s\n", Version) + gologger.Info().Msgf("Current Version: %s\n", Version) os.Exit(0) } @@ -111,7 +111,7 @@ func ParseOptions() *Options { // invalid options have been used, exit. err = options.validateOptions() if err != nil { - gologger.Fatalf("Program exiting: %s\n", err) + gologger.Fatal().Msgf("Program exiting: %s\n", err) } return options @@ -130,9 +130,9 @@ func hasStdin() bool { } func listSources(options *Options) { - gologger.Infof("Current list of available sources. [%d]\n", len(options.YAMLConfig.AllSources)) - gologger.Infof("Sources marked with an * needs key or token in order to work.\n") - gologger.Infof("You can modify %s to configure your keys / tokens.\n\n", options.ConfigFile) + gologger.Info().Msgf("Current list of available sources. [%d]\n", len(options.YAMLConfig.AllSources)) + gologger.Info().Msgf("Sources marked with an * needs key or token in order to work.\n") + gologger.Info().Msgf("You can modify %s to configure your keys / tokens.\n\n", options.ConfigFile) keys := options.YAMLConfig.GetKeys() needsKey := make(map[string]interface{}) @@ -146,6 +146,6 @@ func listSources(options *Options) { if _, ok := needsKey[source]; ok { message = "%s *\n" } - gologger.Silentf(message, source) + gologger.Silent().Msgf(message, source) } } diff --git a/v2/pkg/runner/validate.go b/v2/pkg/runner/validate.go index 25d58be..1f833ca 100644 --- a/v2/pkg/runner/validate.go +++ b/v2/pkg/runner/validate.go @@ -4,6 +4,8 @@ import ( "errors" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/formatter" + "github.com/projectdiscovery/gologger/levels" ) // validateOptions validates the configuration options passed @@ -39,12 +41,12 @@ func (options *Options) validateOptions() error { func (options *Options) configureOutput() { // If the user desires verbose output, show verbose output if options.Verbose { - gologger.MaxLevel = gologger.Verbose + gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) } if options.NoColor { - gologger.UseColors = false + gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true)) } if options.Silent { - gologger.MaxLevel = gologger.Silent + gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) } } diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index 31cff36..b4c89ec 100755 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -90,7 +90,7 @@ func (s *Session) DiscardHTTPResponse(response *http.Response) { if response != nil { _, err := io.Copy(ioutil.Discard, response.Body) if err != nil { - gologger.Warningf("Could not discard response body: %s\n", err) + gologger.Warning().Msgf("Could not discard response body: %s\n", err) return } response.Body.Close() diff --git a/v2/pkg/subscraping/sources/cebaidu/cebaidu.go b/v2/pkg/subscraping/sources/cebaidu/cebaidu.go deleted file mode 100644 index 1366cc7..0000000 --- a/v2/pkg/subscraping/sources/cebaidu/cebaidu.go +++ /dev/null @@ -1,63 +0,0 @@ -package cebaidu - -import ( - "context" - "fmt" - - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" -) - -// Source is the passive scraping agent -type Source struct{} - -type domain struct { - Domain string `json:"domain"` -} - -type cebaiduResponse struct { - Code int64 `json:"code"` - Message string `json:"message"` - Data []domain `json:"data"` -} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - defer close(results) - - resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://ce.baidu.com/index/getRelatedSites?site_address=%s", domain)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHTTPResponse(resp) - return - } - - var response cebaiduResponse - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - return - } - resp.Body.Close() - - if response.Code > 0 { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%d, %s", response.Code, response.Message)} - return - } - - for _, result := range response.Data { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Domain} - } - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "cebaidu" -} diff --git a/v2/pkg/subscraping/sources/certspotterold/certspotterold.go b/v2/pkg/subscraping/sources/certspotterold/certspotterold.go deleted file mode 100755 index 4abed9b..0000000 --- a/v2/pkg/subscraping/sources/certspotterold/certspotterold.go +++ /dev/null @@ -1,75 +0,0 @@ -package certspotterold - -import ( - "context" - "fmt" - "net/http" - - jsoniter "github.com/json-iterator/go" - - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" -) - -type errorResponse struct { - Code string `json:"code"` - Message string `json:"Message"` -} - -type subdomain struct { - DNSNames []string `json:"dns_names"` -} - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - defer close(results) - - resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://certspotter.com/api/v0/certs?domain=%s", domain)) - if err != nil && resp == nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHTTPResponse(resp) - return - } - - if resp.StatusCode != http.StatusOK { - var errResponse errorResponse - err = jsoniter.NewDecoder(resp.Body).Decode(&errResponse) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - return - } - - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s: %s", errResponse.Code, errResponse.Message)} - resp.Body.Close() - return - } - - var subdomains []subdomain - err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - return - } - - resp.Body.Close() - - for _, subdomain := range subdomains { - for _, dnsname := range subdomain.DNSNames { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: dnsname} - } - } - }() - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "certspotterold" -} diff --git a/v2/pkg/subscraping/sources/github/github.go b/v2/pkg/subscraping/sources/github/github.go index d384de8..e0d42ba 100644 --- a/v2/pkg/subscraping/sources/github/github.go +++ b/v2/pkg/subscraping/sources/github/github.go @@ -69,7 +69,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * if token.RetryAfter > 0 { if len(tokens.pool) == 1 { - gologger.Verbosef("GitHub Search request rate limit exceeded, waiting for %d seconds before retry... \n", s.Name(), token.RetryAfter) + gologger.Verbose().Msgf("GitHub Search request rate limit exceeded, waiting for %d seconds before retry... \n", s.Name(), token.RetryAfter) time.Sleep(time.Duration(token.RetryAfter) * time.Second) } else { token = tokens.Get() diff --git a/v2/pkg/subscraping/sources/passivetotal/passivetotal.go b/v2/pkg/subscraping/sources/passivetotal/passivetotal.go index 76495a3..1c7c7ae 100755 --- a/v2/pkg/subscraping/sources/passivetotal/passivetotal.go +++ b/v2/pkg/subscraping/sources/passivetotal/passivetotal.go @@ -3,6 +3,7 @@ package passivetotal import ( "bytes" "context" + "regexp" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" @@ -54,6 +55,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se resp.Body.Close() for _, subdomain := range data.Subdomains { + // skip entries like xxx.xxx.xxx.xxx\032domain.tld + if passiveTotalFilterRegex.MatchString(subdomain) { + continue + } finalSubdomain := subdomain + "." + domain results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: finalSubdomain} } @@ -66,3 +71,5 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se func (s *Source) Name() string { return "passivetotal" } + +var passiveTotalFilterRegex *regexp.Regexp = regexp.MustCompile(`^(?:\d{1,3}\.){3}\d{1,3}\\032`) diff --git a/v2/pkg/subscraping/sources/shodan/shodan.go b/v2/pkg/subscraping/sources/shodan/shodan.go index 31c68f7..0ccb624 100644 --- a/v2/pkg/subscraping/sources/shodan/shodan.go +++ b/v2/pkg/subscraping/sources/shodan/shodan.go @@ -12,14 +12,10 @@ import ( type Source struct{} type dnsdbLookupResponse struct { - Domain string `json:"domain"` - Data []struct { - Subdomain string `json:"subdomain"` - Type string `json:"type"` - Value string `json:"value"` - } `json:"data"` - Result int `json:"result"` - Error string `json:"error"` + Domain string `json:"domain"` + Subdomains []string `json:"subdomains"` + Result int `json:"result"` + Error string `json:"error"` } // Run function returns all subdomains found with the service @@ -54,14 +50,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - for _, data := range response.Data { - if data.Subdomain != "" { - if data.Type == "CNAME" { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Value} - } else if data.Type == "A" { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", data.Subdomain, domain)} - } - } + for _, data := range response.Subdomains { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", data, domain)} } }()