ci: use composite actions

Signed-off-by: Dwi Siswanto <git@dw1.io>

.github/scripts/weak-matcher-checks.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -uo pipefail
+
+OUTPUT="/tmp/nuclei-result-${GITHUB_SHA}.out"
+CHANGED_FILES="$(sed 's/ / -t /g' <<< "${CHANGED_FILES}")"
+WEAK=false
+COMMENT=""
+
+eval "nuclei -duc -silent -ud ${GITHUB_WORKSPACE} -u ${HONEYPOT_URL} -o ${OUTPUT} -t ${CHANGED_FILES}"
+
+if [[ "$(wc -l < $OUTPUT)" -gt 0 ]]; then
+	COMMENT+="**:warning: Weak matcher detected**\n\n"
+	COMMENT+="It looks like Nuclei has found some results on the honeypot target.\n\n"
+	COMMENT+="To improve the accuracy of these results and avoid any false positives, "
+	COMMENT+="please adjust the matchers as needed. "
+	COMMENT+="This will help in providing more reliable and precise results.\n\n"
+	COMMENT+="| **Template ID** |\n"
+	COMMENT+="|--|\n"
+	COMMENT+=$(grep -Po "^\\K[[\w_-]+\]" $OUTPUT | sed 's/\[/| /g; s/\]/ |/g' | sed ':a;N;$!ba;s/\n/\\n/g')
+	COMMENT+="\n\n"
+	COMMENT+="> Ref ${GITHUB_SHA}"
+
+	WEAK=true
+fi
+
+echo "weak=${WEAK}" >> $GITHUB_OUTPUT
+
+{
+	echo "comment<<EOF"
+	echo -e "${COMMENT}"
+	echo "EOF"
+} >> $GITHUB_OUTPUT

.github/scripts/yaml2json/go.mod

@@ -0,0 +1,5 @@
+module yaml2json
+
+go 1.22.0
+
+require gopkg.in/yaml.v3 v3.0.1 // indirect

.github/scripts/yaml2json/go.sum

@@ -0,0 +1,3 @@
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

.github/workflows/autoassign.yml

@@ -7,25 +7,20 @@ on:
   issues:
     types: [opened]
 
+env:
+  ASSIGN_TASK_TOKEN: ${{ secrets.PDTEAMX_PAT }} # github personal token
+
 jobs:
   build:
     permissions: write-all
     runs-on: ubuntu-latest
-    env:
-      ASSIGN_TASK_TOKEN: ${{ secrets.PDTEAMX_PAT }} # github personal token
     steps:
-      - name: checkout repo content
+      - uses: actions/checkout@v4 # checkout the repository content
-        uses: actions/checkout@v4 # checkout the repository content
+      - uses: actions/setup-python@v5
-      - name: setup python
-        uses: actions/setup-python@v5
         with:
           python-version: '3.10' # install the python version needed
-      - name: install python packages
+      - run: pip install requests
-        run: |
+      - if: github.event_name == 'pull_request'
-          pip install requests
-      - name: execute python script on pr
-        if: ${{ github.event_name == 'pull_request' }}
         run: python .github/scripts/assign_tasks.py ${{ github.event.pull_request.number }} pr ${{ secrets.GITHUB_TOKEN }}
-      - name: execute python script on issue opened
+      - if: github.event_name == 'issues'
-        if: ${{ github.event_name == 'issues' }}
         run: python .github/scripts/assign_tasks.py ${{ github.event.issue.number }} issue ${{ secrets.GITHUB_TOKEN }}

.github/workflows/cve2json.yml

@@ -5,41 +5,27 @@ on:
     branches:
       - main
     paths:
-      - '*/cves/**'
+      - '**/cves/**'
-  workflow_dispatch: # allows manual triggering of the workflow
+  workflow_dispatch:
 
 jobs:
   cve2json:
     runs-on: ubuntu-latest
     if: github.repository == 'projectdiscovery/nuclei-templates'
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v4
-      - name: Set up Go
+      - uses: projectdiscovery/actions/setup/go@v1
-        uses: actions/setup-go@v5
         with:
-          go-version: 1.19
+          go-version: 'stable'
-          
+      - run: go run main.go $GITHUB_WORKSPACE/http/cves/,$GITHUB_WORKSPACE/network/cves/ $GITHUB_WORKSPACE/cves.json
-      - name: Run YAML2JSON
+        working-directory: .github/scripts/yaml2json
-        id: cves
+      - run: md5sum cves.json | cut -d' ' -f1 > cves.json-checksum.txt
-        run: |
+      - uses: projectdiscovery/actions/setup/git@v1
-          go env -w GO111MODULE=off
+      - uses: projectdiscovery/actions/commit@v1
-          go get gopkg.in/yaml.v3
+        with:
-          go run .github/scripts/yaml2json.go $GITHUB_WORKSPACE/http/cves/,$GITHUB_WORKSPACE/network/cves/ cves.json
+          files: 'cves.json*'
-          md5sum cves.json | cut -d' ' -f1 > cves.json-checksum.txt
+          message: 'chore: generate CVEs metadata 🤖'
-          git status -s | wc -l | xargs -I {} echo CHANGES={} >> $GITHUB_OUTPUT
-          
-      - name: Commit files
-        if: steps.cves.outputs.CHANGES > 0
-        run: |
-          git config --local user.email "action@github.com"
-          git config --local user.name "GitHub Action"
-          git add cves.json cves.json-checksum.txt
-          git commit -m "Auto Generated cves.json [$(date)] :robot:" -a
-
       - name: Push changes
-        if: steps.cves.outputs.CHANGES > 0
         run: |
           git pull --rebase
-          git push origin ${{ github.ref }}
+          git push origin $GITHUB_REF
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/new-templates.yml

@@ -10,36 +10,29 @@ on:
   release:
     types: [published]  
 
+env:
+  NEW_ADDITION_FILE: '.new-additions'
+
 jobs:
-  templates:
+  new-addition:
     runs-on: ubuntu-latest
     if: github.repository == 'projectdiscovery/nuclei-templates'
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v4
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          persist-credentials: false
           fetch-depth: 0
-
+      - name: Generate new addition list
-      - name: Generate new template list
-        id: new-additions
         run: |
           git pull
-          git diff  --name-only --diff-filter=A $(git tag | tail -n 1) @ . | grep .yaml | tee .new-additions
+          git diff --name-only --diff-filter=A $(git tag | tail -n 1) @ . | grep .yaml | tee $NEW_ADDITION_FILE
-
+      - uses: projectdiscovery/actions/setup/git@v1
-      - name: Commit files
+      - uses: projectdiscovery/actions/commit@v1
-        run: |
-          git config --local user.email "bot@projectdiscovery.io"
-          git config --local user.name "[PDBot]"
-          git add .new-additions -f
-          git commit --allow-empty -m "Auto Generated New Template Addition List [$(date)] :robot:" -a
-
-      - name: Push changes
-        uses: ad-m/github-push-action@master
         with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
+          files: '${{ env.NEW_ADDITION_FILE }}'
-          branch: ${{ github.ref }}
+          commit: 'chore: generate new addition list 🤖'
+      - run: git push origin $GITHUB_REF
 
-      - name: Trigger Remote Workflow with curl
+  sync:
-        run: |
+    needs: new-addition
-          curl -i -s -k -X 'POST' -H 'Host: api.github.com' -H "Authorization: token ${{ secrets.GTOKEN }}" --data-binary $'{\"ref\":\"main\"}' 'https://api.github.com/repos/projectdiscovery/early-templates/actions/workflows/reposync.yml/dispatches'
+    uses: ./.github/workflows/templates-sync.yaml
+    secrets: inherit

.github/workflows/readme-update.yml

@@ -12,36 +12,15 @@ jobs:
     runs-on: ubuntu-latest
     if: github.repository == 'projectdiscovery/nuclei-templates'
     steps:
-      - name: Install tree
+      - run: sudo apt-get install tree -y
-        run: sudo apt-get install tree -y
+      - uses: actions/checkout@v4
-
+      - uses: actions/setup-python@v5
-      - name: Check out code
-        uses: actions/checkout@master
-        with:
-          persist-credentials: false
-          fetch-depth: 0
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
         with:
           python-version: "3.8"
-
+      - run: python .github/scripts/update-readme.py
-      - name: Update README
+      - uses: projectdiscovery/actions/setup/git@v5
-        id: readme-update
+      - uses: projectdiscovery/actions/commit@v5
-        run: |
-          python .github/scripts/update-readme.py
-          git status -s | wc -l | xargs -I {} echo CHANGES={} >> $GITHUB_OUTPUT
-
-      - name: Commit files
-        if: steps.readme-update.outputs.CHANGES > 0
-        run: |
-          git config --local user.email "action@github.com"
-          git config --local user.name "GitHub Action"
-          git commit -m "Auto README Update [$(date)] :robot:" -a
-
-      - name: Push changes
-        if: steps.readme-update.outputs.CHANGES > 0
-        uses: ad-m/github-push-action@master
         with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
+          files: 'README.md'
-          branch: ${{ github.ref }}
+          message: 'docs: update README 🤖'
+      - run: git push origin $GITHUB_REF

.github/workflows/syntax-checking.yml

@@ -1,20 +0,0 @@
-name: ❄️ YAML Lint
-
-on:
-  pull_request:
-    paths:
-      - '**.yaml'
-  workflow_dispatch:
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    if: github.repository == 'projectdiscovery/nuclei-templates'
-    steps:
-      - uses: actions/checkout@v4
-      - name: Yamllint
-        uses: karancode/yamllint-github-action@v2.1.1
-        with:
-          yamllint_config_filepath: .yamllint
-          yamllint_strict: false
-          yamllint_comment: true

.github/workflows/template-checksum.yml

@@ -6,7 +6,10 @@ on:
       - main
     paths:
       - '**.yaml'
-  workflow_dispatch: # allows manual triggering of the workflow
+  workflow_dispatch:
+
+env:
+  CHECKSUM_FILE: 'templates-checksum.txt'
 
 jobs:
   checksum:
@@ -16,34 +19,16 @@ jobs:
       - uses: actions/checkout@v4
         with: 
           fetch-depth: 0
-
+      - uses: projectdiscovery/actions/generate/checksum@v1
-      - name: Set up Go
-        uses: actions/setup-go@v5
         with:
-          go-version: 1.21.x
+          templates-directory: '${{ github.workspace }}'
-
+          checksum-file: '${{ env.CHECKSUM_FILE }}'
-      - name: install checksum generator
+      - uses: projectdiscovery/actions/setup/git@v1
-        run: |
+      - uses: projectdiscovery/actions/commit@v1
-          go install -v github.com/projectdiscovery/nuclei/v3/cmd/generate-checksum@latest
+        with:
-
+          files: '${{ env.CHECKSUM_FILE }}'
-      - name: generate checksum
+          commit: 'chore: generate templates checksum 🤖'
-        id: checksum
-        run: |
-          generate-checksum /home/runner/work/nuclei-templates/nuclei-templates/ templates-checksum.txt
-          git status -s | wc -l | xargs -I {} echo CHANGES={} >> $GITHUB_OUTPUT
-
-      - name: Commit files
-        if: steps.checksum.outputs.CHANGES > 0
-        run: |
-          git config --local user.email "action@github.com"
-          git config --local user.name "GitHub Action"
-          git add templates-checksum.txt
-          git commit -am "Auto Generated Templates Checksum [$(date)] :robot:"
-
       - name: Push changes
-        if: steps.checksum.outputs.CHANGES > 0
         run: |
           git pull --rebase
-          git push origin ${{ github.ref }}
+          git push origin $GITHUB_REF
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/template-db-indexer.yml

@@ -8,24 +8,14 @@ on:
 
 jobs:
   index:
-    runs-on: ubuntu-latest-16-cores
+    runs-on: ubuntu-latest
     if: github.repository == 'projectdiscovery/nuclei-templates'
     steps:
-      - name: Set up Go
+      - uses: projectdiscovery/actions/generate/db-indexer@v1
-        uses: actions/setup-go@v5
         with:
-          go-version: 1.21.x
+          token: '${{ secrets.ACCESS_TOKEN }}'
-
+          mode: 'templates'
-      - name: Installing Indexer
+          args: '-eft'
-        run: |
-          git config --global url."https://${{ secrets.ACCESS_TOKEN }}@github".insteadOf https://github
-          git clone https://github.com/projectdiscovery/nucleish-api.git
-          cd nucleish-api/cmd/generate-index/
-          go install
-
-      - name: Generate Index
         env:
           AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
           AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}
-        run: |
-          generate-index -mode templates -eft

.github/workflows/template-sign.yml

@@ -14,39 +14,19 @@ jobs:
     if: github.repository == 'projectdiscovery/nuclei-templates'
     steps:
       - uses: actions/checkout@v4
-        with: 
+      - uses: projectdiscovery/actions/setup/nuclei@v1
-          fetch-depth: 0
+      - run: nuclei -lfa -duc -sign -ud $GITHUB_WORKSPACE -t .
-
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: 1.21.x
-
-      - name: nuclei install
-        run: go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
-
-      - name: Template Sign
-        id: sign
-        run: |
-          nuclei -lfa -duc -sign -t /home/runner/work/nuclei-templates/nuclei-templates
-          nuclei -lfa -duc -t /home/runner/work/nuclei-templates/nuclei-templates
-          git status -s | wc -l | xargs -I {} echo CHANGES={} >> $GITHUB_OUTPUT
         env:
           NUCLEI_USER_CERTIFICATE: ${{ secrets.NUCLEI_USER_CERTIFICATE }}
           NUCLEI_USER_PRIVATE_KEY: ${{ secrets.NUCLEI_USER_PRIVATE_KEY }}
-          
+      - uses: projectdiscovery/actions/setup/git@v1
-      - name: Commit files
+      - uses: projectdiscovery/actions/commit@v1
-        if: steps.sign.outputs.CHANGES > 0
+        with:
-        run: |
+          files: |
-          git config --local user.email "action@github.com"
+            cves.json
-          git config --local user.name "GitHub Action"
+            cves.json-checksum.txt
-          git add cves.json cves.json-checksum.txt
+          message: 'chore: sign templates 🤖'
-          git commit -m "Auto Template Signing [$(date)] :robot:" -a
-
       - name: Push changes
-        if: steps.sign.outputs.CHANGES > 0
         run: |
           git pull --rebase
           git push origin ${{ github.ref }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/template-validate.yml

@@ -1,30 +0,0 @@
-name: 🛠 Template Validate
-
-on:
-  pull_request:
-    paths:
-      - '**.yaml'
-  workflow_dispatch:
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    if: github.repository == 'projectdiscovery/nuclei-templates'
-    steps:
-      - uses: actions/checkout@v4
-        with: 
-          fetch-depth: 0
-
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: 1.21.x
-
-      - name: nuclei install
-        run: go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
-
-      - name: Template Validation
-        run: |
-          cp -r ${{ github.workspace }} $HOME
-          nuclei -duc -validate -allow-local-file-access -et /home/runner/nuclei-templates/config/.git -et /home/runner/nuclei-templates/config
-          nuclei -duc -validate -w ./workflows -allow-local-file-access -et /home/runner/nuclei-templates/config/.git -et /home/runner/nuclei-templates/config

.github/workflows/templateman.yml

@@ -11,36 +11,15 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-
+      - uses: projectdiscovery/actions/setup/templateman@v1
-      - name: Set up Go
-        uses: actions/setup-go@v5
         with:
-          go-version: 1.21.x
+          token: '${{ secrets.ACCESS_TOKEN }}'
-
+      - run: tmc -mr -e -at <<< "$(pwd)"
-      - name: Install TemplateMan CLI Client
+      - uses: projectdiscovery/actions/setup/git@v1
-        run: |
+      - uses: projectdiscovery/actions/commit@v1
-          git config --global url."https://${{ secrets.ACCESS_TOKEN }}@github".insteadOf https://github
+        with:
-          git clone https://github.com/projectdiscovery/templateman.git
+          message: 'chore: update TemplateMan 🤖'
-          cd templateman/templateman-cli/cmd/tmc
-          go install
-
-      - name: Run TemplateMan
-        id: tmc
-        run: |
-          echo /home/runner/work/nuclei-templates/nuclei-templates | tmc -mr -e -at
-          git status -s | wc -l | xargs -I {} echo CHANGES={} >> $GITHUB_OUTPUT
-
-      - name: Commit files
-        if: steps.tmc.outputs.CHANGES > 0
-        run: |
-          git config --local user.email "action@github.com"
-          git config --local user.name "GitHub Action"
-          git commit --allow-empty -m "TemplateMan Update [$(date)] :robot:" -a
-
       - name: Push changes
-        if: steps.tmc.outputs.CHANGES > 0
         run: |
           git pull --rebase
-          git push origin ${{ github.ref }}
+          git push origin $GITHUB_REF
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/templates-stats.yml

@@ -14,46 +14,27 @@ jobs:
       - uses: actions/checkout@v4
         with: 
           fetch-depth: 0
-
+      - uses: projectdiscovery/actions/templates/stats@v1
-      - name: Set up Go
-        uses: actions/setup-go@v5
         with:
-          go-version: 1.19
+          path: '${{ github.workspace }}'
-
+          output: 'TEMPLATES-STATS.md'
-      - name: Installing Template Stats
+      - uses: projectdiscovery/actions/templates/stats@v1
-        run: |
+        with:
-          go install github.com/projectdiscovery/templates-stats@main
+          path: '${{ github.workspace }}'
-        shell: bash
+          output: 'TEMPLATES-STATS.json'
-
+          args: '-json'
-      - name: Markdown Stats
+      - uses: projectdiscovery/actions/templates/stats@v1
-        run: |
+        with:
-          templates-stats -output TEMPLATES-STATS.md -path /home/runner/work/nuclei-templates/nuclei-templates/
+          path: '${{ github.workspace }}'
-        shell: bash
+          output: 'TOP-10.md'
-
+          args: '-top 10'
-      - name: JSON Stats
+      - uses: projectdiscovery/actions/setup/git@v1
-        run: |
+      - uses: projectdiscovery/actions/commit@v1
-          templates-stats -output TEMPLATES-STATS.json -json -path /home/runner/work/nuclei-templates/nuclei-templates/
+        with:
-        shell: bash
+          files: |
-
+            TEMPLATES-STATS.*
-      - name: Top 10 Stats
+            TOP-10.md
-        run: |
+          message: 'chore: generate templates stats 🤖'
-          templates-stats -output TOP-10.md -top 10 -path /home/runner/work/nuclei-templates/nuclei-templates/
+      - uses: ad-m/github-push-action@master
-        shell: bash
-
-      - name: Get statistical changes
-        id: stats
-        run: git status -s | wc -l | xargs -I {} echo CHANGES={} >> $GITHUB_OUTPUT
-
-      - name: Commit files
-        if: steps.stats.outputs.CHANGES > 0
-        run: |
-          git add TEMPLATES-STATS.*
-          git add TOP-10.md
-          git config --local user.email "action@github.com"
-          git config --local user.name "GitHub Action"
-          git commit -m "Auto Generated Templates Stats [$(date)] :robot:" -a
-
-      - name: Push changes
-        uses: ad-m/github-push-action@master
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/templates-sync.yml

@@ -1,4 +1,5 @@
 name: Sync Repositories Workflow
+
 on:
   push:
     paths:
@@ -29,11 +30,13 @@ on:
     - 'javascript/misconfiguration/x11/x11-unauth-access.yaml'
     - 'javascript/udp/detection/db2-discover.yaml'
   workflow_dispatch:
+  workflow_call:
+
 jobs:
-  triggerRemoteWorkflow:
+  sync:
     if: github.repository == 'projectdiscovery/nuclei-templates'
     runs-on: ubuntu-latest
     steps:
-    - name: Trigger Remote Workflow with curl
+      - uses: projectdiscovery/actions/templates/sync@v1
-      run: |
+        with:
-        curl -i -s -k -X 'POST' -H 'Host: api.github.com' -H "Authorization: token ${{ secrets.GTOKEN }}" --data-binary $'{\"ref\":\"main\"}' 'https://api.github.com/repos/projectdiscovery/early-templates/actions/workflows/reposync.yml/dispatches'
+          token: '${{ secrets.GTOKEN }}'

.github/workflows/tests.yaml

@@ -0,0 +1,75 @@
+name: 🛠 Tests
+
+on:
+  pull_request:
+    paths:
+      - '**.yaml'
+  workflow_dispatch:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    if: github.repository == 'projectdiscovery/nuclei-templates'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: karancode/yamllint-github-action@v2.1.1
+        with:
+          yamllint_config_filepath: .yamllint
+          yamllint_strict: false
+          yamllint_comment: true
+
+  validate:
+    needs: lint
+    runs-on: ubuntu-latest
+    if: github.repository == 'projectdiscovery/nuclei-templates'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: projectdiscovery/actions/setup/nuclei@v1
+      - name: Validate templates
+        run: nuclei -duc -validate -lfa -ud $GITHUB_WORKSPACE -w workflows/ -et .github/
+
+  weak-matcher-checks:
+    needs: lint
+    runs-on: ubuntu-latest
+    if: github.repository == 'projectdiscovery/nuclei-templates'
+    env:
+      HONEYPOT_URL: 'http://honey.scanme.sh'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          list-files: shell
+          filters: |
+            templates: &templates
+              - '**.yml'
+              - '**.yaml'
+              - '!.github/**'
+            changed:
+              - added|modified: *templates
+      - uses: projectdiscovery/actions/setup/nuclei@v1
+      - run: bash weak-matcher-checks.sh
+        id: check
+        if: steps.filter.outputs.changed == 'true'
+        env:
+          CHANGED_FILES: '${{ steps.filter.outputs.changed_files }}'
+        working-directory: '.github/scripts/'
+      - uses: actions/github-script@v7
+        if: steps.check.outputs.weak == 'true'
+        with:
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: process.env.COMMENT
+            })
+
+            github.rest.issues.addLabels({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              labels: ['false-positive']
+            })
+        env:
+          COMMENT: '${{ steps.check.outputs.comment }}'