id: robots-txt-endpoint

info:
  name: robots.txt endpoint prober
  author: CasperGN,pdteam
  severity: info
  metadata:
    max-request: 2
  tags: misc,generic

http:
  - method: GET
    path:
      - "{{RootURL}}/robots.txt"
      - "{{RootURL}}{{endpoint}}"

    extractors:
      - type: regex
        name: endpoint
        group: 1
        regex:
          - '(?m:\s(/[[:alpha:]]+[[:graph:]]+))'
        internal: true
    iterate-all: true

    matchers-condition: and
    matchers:
      - type: word
        part: body
        words:
          - 'User-agent:'
          - 'Disallow:'
          - 'Allow:'

      - type: word
        part: header
        words:
          - "text/plain"

      - type: status
        status:
          - 200

# digest: 490a0046304402207a390f453c435c533b20adfa0b547d200d75c16ea6fec67753cfd1bcf4c798c2022005a82d5a928b8f08ba54644f69befb25a9f69d8b919539c67136046664a4c76d:922c64590222798bb761d5b6d8e72950