Merge pull request #7451 from projectdiscovery/princechaddha-patch-7

Updated robots-txt-endpoint and robots-txt

http/miscellaneous/robots-txt-endpoint.yaml

@@ -23,7 +23,20 @@ http:
         internal: true
 
     iterate-all: true
+    matchers-condition: and
     matchers:
+      - type: word
+        part: body
+        words:
+          - 'User-agent:'
+          - 'Disallow:'
+          - 'Allow:'
+
+      - type: word
+        part: header
+        words:
+          - "text/plain"
+
       - type: status
         status:
           - 200

http/miscellaneous/robots-txt.yaml

@@ -13,12 +13,15 @@ http:
     path:
       - "{{BaseURL}}/robots.txt"
 
-    matchers-condition: and
     host-redirects: true
+    max-redirects: 2
+    matchers-condition: and
     matchers:
       - type: word
         words:
-          - "Disallow:"
+          - 'User-agent:'
+          - 'Disallow:'
+          - 'Allow:'
 
       - type: word
         part: header