Fix categorized domain filtering logic

.gitignore

@@ -1,6 +1,7 @@
 *.html
 *.txt
 *.jpg
+Pipfile*
 
 .vscode/*
 !.vscode/settings.json

README.md

@@ -8,11 +8,12 @@ This Python based tool was written to quickly query the Expireddomains.net searc
 
 ## Changes
 
+- 5 October 2018
+   + Fixed logic for filtering domains with desirable categorizations. Previously, some error conditions weren't filtered and would result in domains without a valid categorization making it into the final list.
+
 - 4 October 2018
    + Tweaked parsing logic
    + Fixed changes parsed columns indexes
-   + Added additional TLDs to found TLD if the TLD is marked available.
-        + If thisistest.com is found and thisistest.org is mark available, thisistest.org will be added to the search list
 
 - 17 September 2018
     + Fixed Symantec WebPulse Site Review parsing errors caused by service updates

domainhunter.py

@@ -16,7 +16,7 @@ import json
 import base64
 import os
 
-__version__ = "20180917"
+__version__ = "20181005"
 
 ## Functions
 
@@ -86,7 +86,7 @@ def checkBluecoat(domain):
 
     except Exception as e:
         print('[-] Error retrieving Bluecoat reputation! {0}'.format(e))
-        return "-"
+        return "error"
 
 def checkIBMXForce(domain):
     try: 
@@ -122,9 +122,9 @@ def checkIBMXForce(domain):
 
         return a
 
-    except:
+    except Exception as e:
-        print('[-] Error retrieving IBM x-Force reputation!')
+        print('[-] Error retrieving IBM-Xforce reputation! {0}'.format(e))
-        return "-"
+        return "error"
 
 def checkTalos(domain):
     url = 'https://www.talosintelligence.com/sb_api/query_lookup?query=%2Fapi%2Fv2%2Fdetails%2Fdomain%2F&query_entry={0}&offset=0&order=ip+asc'.format(domain)
@@ -150,9 +150,9 @@ def checkTalos(domain):
        
         return a
 
-    except:
+    except Exception as e:
-        print('[-] Error retrieving Talos reputation!')
+        print('[-] Error retrieving Talos reputation! {0}'.format(e))
-        return "-"
+        return "error"
 
 def checkMXToolbox(domain):
     url = 'https://mxtoolbox.com/Public/Tools/BrandReputation.aspx'
@@ -211,7 +211,7 @@ def checkMXToolbox(domain):
 
     except Exception as e:
         print('[-] Error retrieving Google SafeBrowsing and PhishTank reputation!')
-        return "-"
+        return "error"
 
 def downloadMalwareDomains(malwaredomainsURL):
     url = malwaredomainsURL
@@ -403,7 +403,7 @@ If you plan to use this content for illegal purpose, don't.  Have a nice day :)'
         checkDomain(single)
         exit(0)
 
-    # Perform detailed domain reputation checks against input file, print table, and quit
+    # Perform detailed domain reputation checks against input file, print table, and quit. This does not generate an HTML report
     if filename:
         # Initialize our list with an empty row for the header
         data = []
@@ -451,10 +451,10 @@ If you plan to use this content for illegal purpose, don't.  Have a nice day :)'
         print('[*] Fetching expired or deleted domains containing "{}"'.format(keyword))
         for i in range (0,maxresults,25):
             if i == 0:
-                urls.append("{}/?q={}&fwhois=22&falexa={}".format(expireddomainsqueryURL,keyword,alexa))
+                urls.append("{}/?q={}&fwhois=22&ftlds[]=2&ftlds[]=3&ftlds[]=4&falexa={}".format(expireddomainsqueryURL,keyword,alexa))
                 headers['Referer'] ='https://www.expireddomains.net/domain-name-search/?q={}&start=1'.format(keyword)
             else:
-                urls.append("{}/?start={}&q={}&fwhois=22&falexa={}".format(expireddomainsqueryURL,i,keyword,alexa))
+                urls.append("{}/?start={}&q={}&ftlds[]=2&ftlds[]=3&ftlds[]=4&fwhois=22&falexa={}".format(expireddomainsqueryURL,i,keyword,alexa))
                 headers['Referer'] ='https://www.expireddomains.net/domain-name-search/?start={}&q={}'.format((i-25),keyword)
     
     # If no keyword provided, generate list of recently expired domains URLS (batches of 25 results).
@@ -545,25 +545,8 @@ If you plan to use this content for illegal purpose, don't.  Have a nice day :)'
                         # Only add Expired, not Pending, Backorder, etc
                         if c13 == "Expired":
                             # Append parsed domain data to list if it matches our criteria (.com|.net|.org and not a known malware domain)
-                            #if (c0.lower().endswith(".com") or c0.lower().endswith(".net") or c0.lower().endswith(".org")) and (c0 not in maldomainsList):
+                            if (c0.lower().endswith(".com") or c0.lower().endswith(".net") or c0.lower().endswith(".org")) and (c0 not in maldomainsList):
-                            #    domain_list.append([c0,c3,c4,available,status]) 
+                                domain_list.append([c0,c3,c4,available,status]) 
-
-                            # Add other TLDs to list if marked available
-                            if (c7 == "available") and (c0 not in maldomainsList):
-                                dom = c0.split(".")[0] + ".com"
-                                domain_list.append([dom,c3,c4,available,status]) 
-
-                            if (c8 == "available") and (c0 not in maldomainsList):
-                                dom = c0.split(".")[0] + ".net"
-                                domain_list.append([dom,c3,c4,available,status])    
-
-                            if (c9 == "available") and (c0 not in maldomainsList):
-                                dom = c0.split(".")[0] + ".org"
-                                domain_list.append([dom,c3,c4,available,status])  
-
-                            if (c10 == "available") and (c0 not in maldomainsList):
-                                dom = c0.split(".")[0] + ".de"
-                                domain_list.append([dom,c3,c4,available,status])  
 
                     # Non-keyword search table format is slightly different
                     else:
@@ -600,26 +583,9 @@ If you plan to use this content for illegal purpose, don't.  Have a nice day :)'
 
                         status = ""
 
-                        # Add other TLDs to list if marked available
-                        if (c7 == "available") and (c0 not in maldomainsList):
-                            dom = c0.split(".")[0] + ".com"
-                            domain_list.append([dom,c3,c4,available,status]) 
-
-                        if (c8 == "available") and (c0 not in maldomainsList):
-                            dom = c0.split(".")[0] + ".net"
-                            domain_list.append([dom,c3,c4,available,status])    
-                        
-                        if (c9 == "available") and (c0 not in maldomainsList):
-                            dom = c0.split(".")[0] + ".org"
-                            domain_list.append([dom,c3,c4,available,status])  
-                    
-                        if (c10 == "available") and (c0 not in maldomainsList):
-                            dom = c0.split(".")[0] + ".de"
-                            domain_list.append([dom,c3,c4,available,status])  
-
                         # Append original parsed domain data to list if it matches our criteria (.com|.net|.org and not a known malware domain)
-                        #if (c0.lower().endswith(".com") or c0.lower().endswith(".net") or c0.lower().endswith(".org")) and (c0 not in maldomainsList):
+                        if (c0.lower().endswith(".com") or c0.lower().endswith(".net") or c0.lower().endswith(".org")) and (c0 not in maldomainsList):
-                        #    domain_list.append([c0,c3,c4,available,status]) 
+                            domain_list.append([c0,c3,c4,available,status]) 
                         
         except Exception as e: 
             print("[!] Error: ", e)
@@ -633,13 +599,12 @@ If you plan to use this content for illegal purpose, don't.  Have a nice day :)'
         print("[-] No domain results found or none are currently available for purchase!")
         exit(0)
     else:
-        if check:
-            print("\n[*] Performing reputation checks for {} domains".format(len(domain_list)))
-        
         domain_list_unique = []
         [domain_list_unique.append(item) for item in domain_list if item not in domain_list_unique]
 
-        index = 1
+        # Print number of domains to perform reputation checks against
+        if check:
+            print("\n[*] Performing reputation checks for {} domains".format(len(domain_list_unique)))
 
         for domain_entry in domain_list_unique:
             domain = domain_entry[0]
@@ -647,42 +612,38 @@ If you plan to use this content for illegal purpose, don't.  Have a nice day :)'
             archiveentries = domain_entry[2]
             availabletlds = domain_entry[3]
             status = domain_entry[4]
-            bluecoat = ''
+            bluecoat = '-'
-            ibmxforce = ''
+            ibmxforce = '-'
-            ciscotalos = ''
+            ciscotalos = '-'
 
             # Perform domain reputation checks
             if check:
-                print("[*] Domain {} of {}".format(str(index),str(len(domain_list))))
+                
                 bluecoat = checkBluecoat(domain)
                 print("[+] {}: {}".format(domain, bluecoat))
                 ibmxforce = checkIBMXForce(domain)
                 print("[+] {}: {}".format(domain, ibmxforce))
                 ciscotalos = checkTalos(domain)
                 print("[+] {}: {}".format(domain, ciscotalos))
+                print("")
                 # Sleep to avoid captchas
                 doSleep(timing)
 
-            # Mark reputation checks as skipped if -c flag not present
-            else:
-                bluecoat = '-'
-                ibmxforce = '-'
-                ciscotalos = '-'
-
             # Append entry to new list with reputation if at least one service reports reputation
-            if not ((bluecoat in ('Uncategorized','badurl','Suspicious','Malicious Sources/Malnets','captcha','Phishing')) and ibmxforce == "Not found." and ciscotalos == "Uncategorized"):
+            if not ((bluecoat in ('Uncategorized','badurl','Suspicious','Malicious Sources/Malnets','captcha','Phishing','Placeholders','Spam','error')) \
-                data.append([domain,birthdate,archiveentries,availabletlds,status,bluecoat,ibmxforce,ciscotalos])
+                and (ibmxforce in ('Not found.','error')) and (ciscotalos in ('Uncategorized','error'))):
                 
-            index += 1
+                data.append([domain,birthdate,archiveentries,availabletlds,status,bluecoat,ibmxforce,ciscotalos])
 
     # Sort domain list by column 2 (Birth Year)
     sortedDomains = sorted(data, key=lambda x: x[1], reverse=True) 
 
+    if check:
         if len(sortedDomains) == 0:
-        print("\n[-] No domains discovered with a desireable categorization!")
+            print("[-] No domains discovered with a desireable categorization!")
             exit(0)
         else:
-        print("\n[*] {} of {} domains discovered with a potentially desireable categorization!".format(len(sortedDomains),len(domain_list)))
+            print("[*] {} of {} domains discovered with a potentially desireable categorization!".format(len(sortedDomains),len(domain_list)))
 
     # Build HTML Table
     html = ''