[DRAFT] Annotate CVE: add configurations metadata (#3486)

* improve annotate to use new nvd apis

* annotate

* improvements

* fix mod files

* fetch EPSS and only write CPE

* lint fixes

---------

Co-authored-by: Mzack9999 <mzack9999@protonmail.com>
Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
dev
sduc 2023-04-12 12:49:49 +02:00 committed by GitHub
parent ece20ec15c
commit 7029741338
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 130 additions and 18 deletions

View File

@ -5,10 +5,13 @@ import (
"encoding/json"
"flag"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"regexp"
"strconv"
"strings"
"github.com/pkg/errors"
@ -56,10 +59,7 @@ func process() error {
}
defer os.RemoveAll(tempDir)
client, err := nvd.NewClient(tempDir)
if err != nil {
return err
}
client := nvd.NewClientV2()
catalog := disk.NewCatalog(*templateDir)
paths, err := catalog.GetTemplatePath(*input)
@ -109,7 +109,7 @@ var badRefs = []string{
"secunia.com/",
}
func getCVEData(client *nvd.Client, filePath, data string) {
func getCVEData(client *nvd.ClientV2, filePath, data string) {
matches := idRegex.FindAllStringSubmatch(data, 1)
if len(matches) == 0 {
return
@ -142,13 +142,18 @@ func getCVEData(client *nvd.Client, filePath, data string) {
return
}
var cweID []string
for _, problemData := range cveItem.CVE.Problemtype.ProblemtypeData {
for _, description := range problemData.Description {
for _, weaknessData := range cveItem.Cve.Weaknesses {
for _, description := range weaknessData.Description {
cweID = append(cweID, description.Value)
}
}
cvssScore := cveItem.Impact.BaseMetricV3.CvssV3.BaseScore
cvssMetrics := cveItem.Impact.BaseMetricV3.CvssV3.VectorString
cvssData, err := getPrimaryCVSSData(cveItem)
if err != nil {
log.Printf("Could not get CVSS data %s: %s\n", cveName, err)
return
}
cvssScore := cvssData.BaseScore
cvssMetrics := cvssData.VectorString
// Perform some hacky string replacement to place the metadata in templates
infoBlockIndexData := data[strings.Index(data, "info:"):]
@ -191,12 +196,13 @@ func getCVEData(client *nvd.Client, filePath, data string) {
}
}
// If there is no description field, fill the description from CVE information
hasDescriptionData := len(cveItem.CVE.Description.DescriptionData) > 0
enDescription, err := getEnglishLangString(cveItem.Cve.Descriptions)
hasDescriptionData := err != nil
isDescriptionEmpty := infoBlock.Info.Description == ""
if isDescriptionEmpty && hasDescriptionData {
changed = true
// removes all new lines
description := stringsutil.ReplaceAll(cveItem.CVE.Description.DescriptionData[0].Value, "", "\n", "\\", "'", "\t")
description := stringsutil.ReplaceAll(enDescription, "", "\n", "\\", "'", "\t")
description += "\n"
infoBlock.Info.Description = description
}
@ -205,13 +211,13 @@ func getCVEData(client *nvd.Client, filePath, data string) {
var referenceDataURLs []string
// skip sites that are no longer alive
for _, reference := range cveItem.CVE.References.ReferenceData {
for _, reference := range cveItem.Cve.References {
if stringsutil.ContainsAny(reference.URL, badRefs...) {
continue
}
referenceDataURLs = append(referenceDataURLs, reference.URL)
}
hasReferenceData := len(cveItem.CVE.References.ReferenceData) > 0
hasReferenceData := len(cveItem.Cve.References) > 0
areCveReferencesContained := sliceutil.ContainsItems(infoBlock.Info.Reference, referenceDataURLs)
referencesCount := len(infoBlock.Info.Reference)
if hasReferenceData && !areCveReferencesContained {
@ -226,6 +232,36 @@ func getCVEData(client *nvd.Client, filePath, data string) {
infoBlock.Info.Reference = sliceutil.PruneEmptyStrings(sliceutil.Dedupe(infoBlock.Info.Reference))
}
cpeSet := map[string]bool{}
for _, config := range cveItem.Cve.Configurations {
// Right now this covers only simple configurations. More complex configurations can have multiple CPEs
if len(config.Nodes) == 1 {
changed = true
node := config.Nodes[0]
for _, match := range node.CpeMatch {
cpeSet[extractVersionlessCpe((match.Criteria))] = true
}
}
}
uniqueCpes := make([]string, 0, len(cpeSet))
for k := range cpeSet {
uniqueCpes = append(uniqueCpes, k)
}
if len(uniqueCpes) == 1 {
infoBlock.Info.Classification.Cpe = uniqueCpes[0]
}
epss, err := fetchEpss(cveName)
if err != nil {
log.Printf("Could not fetch Epss score: %s\n", err)
return
}
hasEpssChanged := epss != infoBlock.Info.Classification.EpssScore
if hasEpssChanged {
changed = true
infoBlock.Info.Classification.EpssScore = epss
}
var newInfoBlock bytes.Buffer
yamlEncoder := yaml.NewEncoder(&newInfoBlock)
yamlEncoder.SetIndent(yamlIndentSpaces)
@ -243,6 +279,29 @@ func getCVEData(client *nvd.Client, filePath, data string) {
}
}
func getPrimaryCVSSData(vuln nvd.Vulnerability) (nvd.CvssData, error) {
for _, data := range vuln.Cve.Metrics.CvssMetricV31 {
if data.Type == "Primary" {
return data.CvssData, nil
}
}
for _, data := range vuln.Cve.Metrics.CvssMetricV3 {
if data.Type == "Primary" {
return data.CvssData, nil
}
}
return nvd.CvssData{}, fmt.Errorf("no primary cvss metric found")
}
func getEnglishLangString(data []nvd.LangString) (string, error) {
for _, item := range data {
if item.Lang == "en" {
return item.Value, nil
}
}
return "", fmt.Errorf("no english item found")
}
func isSeverityMatchingCvssScore(severity string, score float64) string {
if score == 0.0 {
return ""
@ -264,6 +323,51 @@ func isSeverityMatchingCvssScore(severity string, score float64) string {
return ""
}
func extractVersionlessCpe(cpe string) string {
parts := strings.Split(cpe, ":")
versionlessPart := parts[0:5]
rest := strings.Split(strings.Repeat("*", len(parts)-len(versionlessPart)), "")
return strings.Join(append(versionlessPart, rest...), ":")
}
type ApiFirstEpssResponse struct {
Status string `json:"status"`
StatusCode int `json:"status-code"`
Version string `json:"version"`
Access string `json:"access"`
Total int `json:"total"`
Offset int `json:"offset"`
Limit int `json:"limit"`
Data []struct {
Cve string `json:"cve"`
Epss string `json:"epss"`
Percentile string `json:"percentile"`
Date string `json:"date"`
} `json:"data"`
}
func fetchEpss(cveId string) (float64, error) {
resp, err := http.Get(fmt.Sprintf("https://api.first.org/data/v1/epss?cve=%s", cveId))
if err != nil {
return 0, fmt.Errorf("unable to fetch EPSS data from first.org: %v", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return 0, fmt.Errorf("unable to read reponse body: %v", err)
}
var parsedResp ApiFirstEpssResponse
err = json.Unmarshal(body, &parsedResp)
if err != nil {
return 0, fmt.Errorf("error while parsing EPSS response: %v", err)
}
if len(parsedResp.Data) != 1 {
return 0, fmt.Errorf("unexpected number of results in EPSS response. Expecting exactly 1, got %v", len(parsedResp.Data))
}
epss := parsedResp.Data[0].Epss
return strconv.ParseFloat(epss, 64)
}
type cisaKEVData struct {
Vulnerabilities []struct {
CVEID string `json:"cveID"`
@ -392,6 +496,8 @@ type TemplateClassification struct {
CvssScore float64 `yaml:"cvss-score,omitempty"`
CveId string `yaml:"cve-id,omitempty"`
CweId string `yaml:"cwe-id,omitempty"`
Cpe string `yaml:"cpe,omitempty"`
EpssScore float64 `yaml:"epss-score,omitempty"`
}
type TemplateInfo struct {

View File

@ -74,7 +74,7 @@ require (
github.com/projectdiscovery/gologger v1.1.8
github.com/projectdiscovery/httpx v1.2.9
github.com/projectdiscovery/mapcidr v1.1.1
github.com/projectdiscovery/nvd v1.0.9
github.com/projectdiscovery/nvd v1.0.10-0.20230327073015-721181aba1e8
github.com/projectdiscovery/ratelimit v0.0.6
github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917
github.com/projectdiscovery/sarif v0.0.1

View File

@ -58,6 +58,7 @@ github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy
github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
@ -362,6 +363,7 @@ github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXm
github.com/microcosm-cc/bluemonday v1.0.23 h1:SMZe2IGa0NuHvnVNAZ+6B38gsTbi5e4sViiWJyDDqFY=
github.com/microcosm-cc/bluemonday v1.0.23/go.mod h1:mN70sk7UkkF8TUr2IGBpNN0jAgStuPzlK76QuruE/z4=
github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
github.com/miekg/dns v1.1.53 h1:ZBkuHr5dxHtB1caEOlZTLPo7D3L3TWckgUUs/RHfDxw=
github.com/miekg/dns v1.1.53/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY=
github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
@ -441,8 +443,8 @@ github.com/projectdiscovery/mapcidr v1.1.1 h1:68Xvw9cKugNeAVxHE3Nl1Ej26nm1taWq6e
github.com/projectdiscovery/mapcidr v1.1.1/go.mod h1:yyp9ghqmmC0+r5DySgDBXE4cf2QW8SBloVESCteWiAg=
github.com/projectdiscovery/networkpolicy v0.0.4 h1:zcGjEqZbyECZEdyCy1jVuwOS7Ww1mzgCefQU75XqdJA=
github.com/projectdiscovery/networkpolicy v0.0.4/go.mod h1:DIXwKs3sQyfCoWHKRLQiRrEorSQW4Zrh4ftu7oDVK6w=
github.com/projectdiscovery/nvd v1.0.9 h1:2DdMm7lu3GnCQsyYDEQiQ/LRYDmpEm654kvGQS6jzjE=
github.com/projectdiscovery/nvd v1.0.9/go.mod h1:nGHAo7o6G4V4kscZlm488qKp/ZrZYiBoKqAQrn3X4Og=
github.com/projectdiscovery/nvd v1.0.10-0.20230327073015-721181aba1e8 h1:aDq18tNWbnN5ZM0ADQb+8KB4DEPIGZMXdDmcXyFUoNg=
github.com/projectdiscovery/nvd v1.0.10-0.20230327073015-721181aba1e8/go.mod h1:JiVXOIewstCBMPsO+ZnmI43UXMPJGEE1jwuFVz4ujKM=
github.com/projectdiscovery/ratelimit v0.0.6 h1:SAD2ArdT9F8NmbkAIZpl7DjNnbiXdUQLnMZt5dbVmZ0=
github.com/projectdiscovery/ratelimit v0.0.6/go.mod h1:WFL6gIggPLTwYwDbxqQODuWrz/lcMP2E5ofKSAz3YwI=
github.com/projectdiscovery/rawhttp v0.1.11 h1:NbfunXIVdmFWAhZ864fx09sQLnHOVTYKhAe9P2Cnass=
@ -461,8 +463,7 @@ github.com/projectdiscovery/tlsx v1.0.6 h1:omMbtedk4BjXtauPpB9Y+FQml9cVthOnIxOMK
github.com/projectdiscovery/tlsx v1.0.6/go.mod h1:9PTwYVVbaLYpNIwZIvgVxJzctbiemM/pgukkOb3/4wY=
github.com/projectdiscovery/uncover v1.0.2 h1:mRFzflYyvwKkHd3XKufMlDRrb6p1mjFZTSHoNAUpFwo=
github.com/projectdiscovery/uncover v1.0.2/go.mod h1:lz4QYfArSA6jJkXyB71kN2/Pc7IW7nJB8c95n7xtwqY=
github.com/projectdiscovery/utils v0.0.20-0.20230410124851-595261704707 h1:usIIWZ/didvXCAhJ39ruuUKH1po4gTV7O097B+YGXDM=
github.com/projectdiscovery/utils v0.0.20-0.20230410124851-595261704707/go.mod h1:jOpbC9qq5sAjvxpdhubzNf61Kxx83pYFP+WMjOtUs/o=
github.com/projectdiscovery/utils v0.0.3/go.mod h1:ne3eSlZlUKuhjHr8FfsfGcGteCzxcbJvFBx4VDBCxK0=
github.com/projectdiscovery/utils v0.0.20-0.20230410133604-010edb62cb35 h1:UBOE9Eob1wj7YZ1MGBbtHvc3ptqBHvXxNKRVxTg21Rc=
github.com/projectdiscovery/utils v0.0.20-0.20230410133604-010edb62cb35/go.mod h1:jOpbC9qq5sAjvxpdhubzNf61Kxx83pYFP+WMjOtUs/o=
github.com/projectdiscovery/wappalyzergo v0.0.88 h1:N/1vFlKmc3GJco9rANJdHrxg8jdav/xmnICo8rztmH8=
@ -616,6 +617,7 @@ github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhu
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30=
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE=
github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is=
github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw=
github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 h1:QuLjRpIBjqene8VvB+VhQ4eTcQGCQ7JDuk0/Fp4sLLw=
github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw=
go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
@ -626,6 +628,7 @@ go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0
go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
@ -654,6 +657,7 @@ golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/exp v0.0.0-20221019170559-20944726eadf/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 h1:pVgRXcIictcr+lBQIFeiwuwtDIs4eL21OuM9nyAADmo=
golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@ -677,6 +681,7 @@ golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
@ -770,6 +775,7 @@ golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDq
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4=