Merge pull request #10455 from projectdiscovery/ci/use-composite-actions

ci: use composite actions
main
Sandeep Singh 2024-08-09 01:38:39 +05:30 committed by GitHub
commit 5b40716b32
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 226 additions and 289 deletions

33
.github/scripts/weak-matcher-checks.sh vendored Normal file
View File

@ -0,0 +1,33 @@
#!/bin/bash
set -uo pipefail
OUTPUT="/tmp/nuclei-result-${GITHUB_SHA}.out"
CHANGED_FILES="$(sed 's/ / -t /g' <<< "${CHANGED_FILES}")"
WEAK=false
COMMENT=""
eval "nuclei -duc -silent -ud ${GITHUB_WORKSPACE} -u ${HONEYPOT_URL} -o ${OUTPUT} -t ${CHANGED_FILES}"
if [[ "$(wc -l < $OUTPUT)" -gt 0 ]]; then
COMMENT+="**:warning: Weak matcher detected**\n\n"
COMMENT+="It looks like Nuclei has found some results on the honeypot target.\n\n"
COMMENT+="To improve the accuracy of these results and avoid any false positives, "
COMMENT+="please adjust the matchers as needed. "
COMMENT+="This will help in providing more reliable and precise results.\n\n"
COMMENT+="| **Template ID** |\n"
COMMENT+="|--|\n"
COMMENT+=$(grep -Po "^\\K[[\w_-]+\]" $OUTPUT | sed 's/\[/| /g; s/\]/ |/g' | sed ':a;N;$!ba;s/\n/\\n/g')
COMMENT+="\n\n"
COMMENT+="> Ref ${GITHUB_SHA}"
WEAK=true
fi
echo "weak=${WEAK}" >> $GITHUB_OUTPUT
{
echo "comment<<EOF"
echo -e "${COMMENT}"
echo "EOF"
} >> $GITHUB_OUTPUT

5
.github/scripts/yaml2json/go.mod vendored Normal file
View File

@ -0,0 +1,5 @@
module yaml2json
go 1.22.0
require gopkg.in/yaml.v3 v3.0.1 // indirect

3
.github/scripts/yaml2json/go.sum vendored Normal file
View File

@ -0,0 +1,3 @@
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@ -7,25 +7,20 @@ on:
issues:
types: [opened]
env:
ASSIGN_TASK_TOKEN: ${{ secrets.PDTEAMX_PAT }} # github personal token
jobs:
build:
permissions: write-all
runs-on: ubuntu-latest
env:
ASSIGN_TASK_TOKEN: ${{ secrets.PDTEAMX_PAT }} # github personal token
steps:
- name: checkout repo content
uses: actions/checkout@v4 # checkout the repository content
- name: setup python
uses: actions/setup-python@v5
- uses: actions/checkout@v4 # checkout the repository content
- uses: actions/setup-python@v5
with:
python-version: '3.10' # install the python version needed
- name: install python packages
run: |
pip install requests
- name: execute python script on pr
if: ${{ github.event_name == 'pull_request' }}
- run: pip install requests
- if: github.event_name == 'pull_request'
run: python .github/scripts/assign_tasks.py ${{ github.event.pull_request.number }} pr ${{ secrets.GITHUB_TOKEN }}
- name: execute python script on issue opened
if: ${{ github.event_name == 'issues' }}
- if: github.event_name == 'issues'
run: python .github/scripts/assign_tasks.py ${{ github.event.issue.number }} issue ${{ secrets.GITHUB_TOKEN }}

View File

@ -5,41 +5,27 @@ on:
branches:
- main
paths:
- '*/cves/**'
workflow_dispatch: # allows manual triggering of the workflow
- '**/cves/**'
workflow_dispatch:
jobs:
cve2json:
runs-on: ubuntu-latest
if: github.repository == 'projectdiscovery/nuclei-templates'
steps:
- uses: actions/checkout@master
- name: Set up Go
uses: actions/setup-go@v5
- uses: actions/checkout@v4
- uses: projectdiscovery/actions/setup/go@v1
with:
go-version: 1.19
- name: Run YAML2JSON
id: cves
run: |
go env -w GO111MODULE=off
go get gopkg.in/yaml.v3
go run .github/scripts/yaml2json.go $GITHUB_WORKSPACE/http/cves/,$GITHUB_WORKSPACE/network/cves/ cves.json
md5sum cves.json | cut -d' ' -f1 > cves.json-checksum.txt
git status -s | wc -l | xargs -I {} echo CHANGES={} >> $GITHUB_OUTPUT
- name: Commit files
if: steps.cves.outputs.CHANGES > 0
run: |
git config --local user.email "action@github.com"
git config --local user.name "GitHub Action"
git add cves.json cves.json-checksum.txt
git commit -m "Auto Generated cves.json [$(date)] :robot:" -a
go-version: 'stable'
- run: go run main.go $GITHUB_WORKSPACE/http/cves/,$GITHUB_WORKSPACE/network/cves/ $GITHUB_WORKSPACE/cves.json
working-directory: .github/scripts/yaml2json
- run: md5sum cves.json | cut -d' ' -f1 > cves.json-checksum.txt
- uses: projectdiscovery/actions/setup/git@v1
- uses: projectdiscovery/actions/commit@v1
with:
files: 'cves.json*'
message: 'chore: generate CVEs metadata 🤖'
- name: Push changes
if: steps.cves.outputs.CHANGES > 0
run: |
git pull --rebase
git push origin ${{ github.ref }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
git push origin $GITHUB_REF

View File

@ -10,36 +10,29 @@ on:
release:
types: [published]
env:
NEW_ADDITION_FILE: '.new-additions'
jobs:
templates:
new-addition:
runs-on: ubuntu-latest
if: github.repository == 'projectdiscovery/nuclei-templates'
steps:
- uses: actions/checkout@master
- uses: actions/checkout@v4
with:
token: ${{ secrets.GITHUB_TOKEN }}
persist-credentials: false
fetch-depth: 0
- name: Generate new template list
id: new-additions
- name: Generate new addition list
run: |
git pull
git diff --name-only --diff-filter=A $(git tag | tail -n 1) @ . | grep .yaml | tee .new-additions
- name: Commit files
run: |
git config --local user.email "bot@projectdiscovery.io"
git config --local user.name "[PDBot]"
git add .new-additions -f
git commit --allow-empty -m "Auto Generated New Template Addition List [$(date)] :robot:" -a
- name: Push changes
uses: ad-m/github-push-action@master
git diff --name-only --diff-filter=A $(git tag | tail -n 1) @ . | grep .yaml | tee $NEW_ADDITION_FILE
- uses: projectdiscovery/actions/setup/git@v1
- uses: projectdiscovery/actions/commit@v1
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
branch: ${{ github.ref }}
files: '${{ env.NEW_ADDITION_FILE }}'
commit: 'chore: generate new addition list 🤖'
- run: git push origin $GITHUB_REF
- name: Trigger Remote Workflow with curl
run: |
curl -i -s -k -X 'POST' -H 'Host: api.github.com' -H "Authorization: token ${{ secrets.GTOKEN }}" --data-binary $'{\"ref\":\"main\"}' 'https://api.github.com/repos/projectdiscovery/early-templates/actions/workflows/reposync.yml/dispatches'
sync:
needs: new-addition
uses: ./.github/workflows/templates-sync.yaml
secrets: inherit

View File

@ -12,36 +12,15 @@ jobs:
runs-on: ubuntu-latest
if: github.repository == 'projectdiscovery/nuclei-templates'
steps:
- name: Install tree
run: sudo apt-get install tree -y
- name: Check out code
uses: actions/checkout@master
with:
persist-credentials: false
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v5
- run: sudo apt-get install tree -y
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.8"
- name: Update README
id: readme-update
run: |
python .github/scripts/update-readme.py
git status -s | wc -l | xargs -I {} echo CHANGES={} >> $GITHUB_OUTPUT
- name: Commit files
if: steps.readme-update.outputs.CHANGES > 0
run: |
git config --local user.email "action@github.com"
git config --local user.name "GitHub Action"
git commit -m "Auto README Update [$(date)] :robot:" -a
- name: Push changes
if: steps.readme-update.outputs.CHANGES > 0
uses: ad-m/github-push-action@master
- run: python .github/scripts/update-readme.py
- uses: projectdiscovery/actions/setup/git@v5
- uses: projectdiscovery/actions/commit@v5
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
branch: ${{ github.ref }}
files: 'README.md'
message: 'docs: update README 🤖'
- run: git push origin $GITHUB_REF

View File

@ -1,20 +0,0 @@
name: ❄️ YAML Lint
on:
pull_request:
paths:
- '**.yaml'
workflow_dispatch:
jobs:
build:
runs-on: ubuntu-latest
if: github.repository == 'projectdiscovery/nuclei-templates'
steps:
- uses: actions/checkout@v4
- name: Yamllint
uses: karancode/yamllint-github-action@v2.1.1
with:
yamllint_config_filepath: .yamllint
yamllint_strict: false
yamllint_comment: true

View File

@ -6,7 +6,10 @@ on:
- main
paths:
- '**.yaml'
workflow_dispatch: # allows manual triggering of the workflow
workflow_dispatch:
env:
CHECKSUM_FILE: 'templates-checksum.txt'
jobs:
checksum:
@ -16,34 +19,16 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Go
uses: actions/setup-go@v5
- uses: projectdiscovery/actions/generate/checksum@v1
with:
go-version: 1.21.x
- name: install checksum generator
run: |
go install -v github.com/projectdiscovery/nuclei/v3/cmd/generate-checksum@latest
- name: generate checksum
id: checksum
run: |
generate-checksum /home/runner/work/nuclei-templates/nuclei-templates/ templates-checksum.txt
git status -s | wc -l | xargs -I {} echo CHANGES={} >> $GITHUB_OUTPUT
- name: Commit files
if: steps.checksum.outputs.CHANGES > 0
run: |
git config --local user.email "action@github.com"
git config --local user.name "GitHub Action"
git add templates-checksum.txt
git commit -am "Auto Generated Templates Checksum [$(date)] :robot:"
templates-directory: '${{ github.workspace }}'
checksum-file: '${{ env.CHECKSUM_FILE }}'
- uses: projectdiscovery/actions/setup/git@v1
- uses: projectdiscovery/actions/commit@v1
with:
files: '${{ env.CHECKSUM_FILE }}'
commit: 'chore: generate templates checksum 🤖'
- name: Push changes
if: steps.checksum.outputs.CHANGES > 0
run: |
git pull --rebase
git push origin ${{ github.ref }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
git push origin $GITHUB_REF

View File

@ -8,24 +8,14 @@ on:
jobs:
index:
runs-on: ubuntu-latest-16-cores
runs-on: ubuntu-latest
if: github.repository == 'projectdiscovery/nuclei-templates'
steps:
- name: Set up Go
uses: actions/setup-go@v5
- uses: projectdiscovery/actions/generate/db-indexer@v1
with:
go-version: 1.21.x
- name: Installing Indexer
run: |
git config --global url."https://${{ secrets.ACCESS_TOKEN }}@github".insteadOf https://github
git clone https://github.com/projectdiscovery/nucleish-api.git
cd nucleish-api/cmd/generate-index/
go install
- name: Generate Index
token: '${{ secrets.ACCESS_TOKEN }}'
mode: 'templates'
args: '-eft'
env:
AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}
run: |
generate-index -mode templates -eft

View File

@ -14,39 +14,19 @@ jobs:
if: github.repository == 'projectdiscovery/nuclei-templates'
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: 1.21.x
- name: nuclei install
run: go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
- name: Template Sign
id: sign
run: |
nuclei -lfa -duc -sign -t /home/runner/work/nuclei-templates/nuclei-templates
nuclei -lfa -duc -t /home/runner/work/nuclei-templates/nuclei-templates
git status -s | wc -l | xargs -I {} echo CHANGES={} >> $GITHUB_OUTPUT
- uses: projectdiscovery/actions/setup/nuclei@v1
- run: nuclei -lfa -duc -sign -ud $GITHUB_WORKSPACE -t .
env:
NUCLEI_USER_CERTIFICATE: ${{ secrets.NUCLEI_USER_CERTIFICATE }}
NUCLEI_USER_PRIVATE_KEY: ${{ secrets.NUCLEI_USER_PRIVATE_KEY }}
- name: Commit files
if: steps.sign.outputs.CHANGES > 0
run: |
git config --local user.email "action@github.com"
git config --local user.name "GitHub Action"
git add cves.json cves.json-checksum.txt
git commit -m "Auto Template Signing [$(date)] :robot:" -a
- uses: projectdiscovery/actions/setup/git@v1
- uses: projectdiscovery/actions/commit@v1
with:
files: |
cves.json
cves.json-checksum.txt
message: 'chore: sign templates 🤖'
- name: Push changes
if: steps.sign.outputs.CHANGES > 0
run: |
git pull --rebase
git push origin ${{ github.ref }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

View File

@ -1,30 +0,0 @@
name: 🛠 Template Validate
on:
pull_request:
paths:
- '**.yaml'
workflow_dispatch:
jobs:
build:
runs-on: ubuntu-latest
if: github.repository == 'projectdiscovery/nuclei-templates'
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: 1.21.x
- name: nuclei install
run: go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
- name: Template Validation
run: |
cp -r ${{ github.workspace }} $HOME
nuclei -duc -validate -allow-local-file-access -et /home/runner/nuclei-templates/config/.git -et /home/runner/nuclei-templates/config
nuclei -duc -validate -w ./workflows -allow-local-file-access -et /home/runner/nuclei-templates/config/.git -et /home/runner/nuclei-templates/config

View File

@ -11,36 +11,15 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Go
uses: actions/setup-go@v5
- uses: projectdiscovery/actions/setup/templateman@v1
with:
go-version: 1.21.x
- name: Install TemplateMan CLI Client
run: |
git config --global url."https://${{ secrets.ACCESS_TOKEN }}@github".insteadOf https://github
git clone https://github.com/projectdiscovery/templateman.git
cd templateman/templateman-cli/cmd/tmc
go install
- name: Run TemplateMan
id: tmc
run: |
echo /home/runner/work/nuclei-templates/nuclei-templates | tmc -mr -e -at
git status -s | wc -l | xargs -I {} echo CHANGES={} >> $GITHUB_OUTPUT
- name: Commit files
if: steps.tmc.outputs.CHANGES > 0
run: |
git config --local user.email "action@github.com"
git config --local user.name "GitHub Action"
git commit --allow-empty -m "TemplateMan Update [$(date)] :robot:" -a
token: '${{ secrets.ACCESS_TOKEN }}'
- run: tmc -mr -e -at <<< "$(pwd)"
- uses: projectdiscovery/actions/setup/git@v1
- uses: projectdiscovery/actions/commit@v1
with:
message: 'chore: update TemplateMan 🤖'
- name: Push changes
if: steps.tmc.outputs.CHANGES > 0
run: |
git pull --rebase
git push origin ${{ github.ref }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
git push origin $GITHUB_REF

View File

@ -14,46 +14,27 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Go
uses: actions/setup-go@v5
- uses: projectdiscovery/actions/templates/stats@v1
with:
go-version: 1.19
- name: Installing Template Stats
run: |
go install github.com/projectdiscovery/templates-stats@main
shell: bash
- name: Markdown Stats
run: |
templates-stats -output TEMPLATES-STATS.md -path /home/runner/work/nuclei-templates/nuclei-templates/
shell: bash
- name: JSON Stats
run: |
templates-stats -output TEMPLATES-STATS.json -json -path /home/runner/work/nuclei-templates/nuclei-templates/
shell: bash
- name: Top 10 Stats
run: |
templates-stats -output TOP-10.md -top 10 -path /home/runner/work/nuclei-templates/nuclei-templates/
shell: bash
- name: Get statistical changes
id: stats
run: git status -s | wc -l | xargs -I {} echo CHANGES={} >> $GITHUB_OUTPUT
- name: Commit files
if: steps.stats.outputs.CHANGES > 0
run: |
git add TEMPLATES-STATS.*
git add TOP-10.md
git config --local user.email "action@github.com"
git config --local user.name "GitHub Action"
git commit -m "Auto Generated Templates Stats [$(date)] :robot:" -a
- name: Push changes
uses: ad-m/github-push-action@master
path: '${{ github.workspace }}'
output: 'TEMPLATES-STATS.md'
- uses: projectdiscovery/actions/templates/stats@v1
with:
path: '${{ github.workspace }}'
output: 'TEMPLATES-STATS.json'
args: '-json'
- uses: projectdiscovery/actions/templates/stats@v1
with:
path: '${{ github.workspace }}'
output: 'TOP-10.md'
args: '-top 10'
- uses: projectdiscovery/actions/setup/git@v1
- uses: projectdiscovery/actions/commit@v1
with:
files: |
TEMPLATES-STATS.*
TOP-10.md
message: 'chore: generate templates stats 🤖'
- uses: ad-m/github-push-action@master
with:
github_token: ${{ secrets.GITHUB_TOKEN }}

View File

@ -1,4 +1,5 @@
name: Sync Repositories Workflow
on:
push:
paths:
@ -39,11 +40,13 @@ on:
- 'javascript/misconfiguration/x11/x11-unauth-access.yaml'
- 'javascript/udp/detection/db2-discover.yaml'
workflow_dispatch:
workflow_call:
jobs:
triggerRemoteWorkflow:
sync:
if: github.repository == 'projectdiscovery/nuclei-templates'
runs-on: ubuntu-latest
steps:
- name: Trigger Remote Workflow with curl
run: |
curl -i -s -k -X 'POST' -H 'Host: api.github.com' -H "Authorization: token ${{ secrets.GTOKEN }}" --data-binary $'{\"ref\":\"main\"}' 'https://api.github.com/repos/projectdiscovery/early-templates/actions/workflows/reposync.yml/dispatches'
- uses: projectdiscovery/actions/templates/sync@v1
with:
token: '${{ secrets.GTOKEN }}'

75
.github/workflows/tests.yaml vendored Normal file
View File

@ -0,0 +1,75 @@
name: 🛠 Tests
on:
pull_request:
paths:
- '**.yaml'
workflow_dispatch:
jobs:
lint:
runs-on: ubuntu-latest
if: github.repository == 'projectdiscovery/nuclei-templates'
steps:
- uses: actions/checkout@v4
- uses: karancode/yamllint-github-action@v2.1.1
with:
yamllint_config_filepath: .yamllint
yamllint_strict: false
yamllint_comment: true
validate:
needs: lint
runs-on: ubuntu-latest
if: github.repository == 'projectdiscovery/nuclei-templates'
steps:
- uses: actions/checkout@v4
- uses: projectdiscovery/actions/setup/nuclei@v1
- name: Validate templates
run: nuclei -duc -validate -lfa -ud $GITHUB_WORKSPACE -w workflows/ -et .github/
weak-matcher-checks:
needs: lint
runs-on: ubuntu-latest
if: github.repository == 'projectdiscovery/nuclei-templates'
env:
HONEYPOT_URL: 'http://honey.scanme.sh'
steps:
- uses: actions/checkout@v4
- uses: dorny/paths-filter@v3
id: filter
with:
list-files: shell
filters: |
templates: &templates
- '**.yml'
- '**.yaml'
- '!.github/**'
changed:
- added|modified: *templates
- uses: projectdiscovery/actions/setup/nuclei@v1
- run: bash weak-matcher-checks.sh
id: check
if: steps.filter.outputs.changed == 'true'
env:
CHANGED_FILES: '${{ steps.filter.outputs.changed_files }}'
working-directory: '.github/scripts/'
- uses: actions/github-script@v7
if: steps.check.outputs.weak == 'true'
with:
script: |
github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: process.env.COMMENT
})
github.rest.issues.addLabels({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
labels: ['false-positive']
})
env:
COMMENT: '${{ steps.check.outputs.comment }}'