remove debug code
parent
0c687fdad4
commit
e7847ae349
|
@ -192,7 +192,6 @@ class BaseScraper(object):
|
|||
self.set('rights_url', link['href'])
|
||||
|
||||
def scrape_sitemap(url, maxnum=None):
|
||||
print type(maxnum)
|
||||
try:
|
||||
response = requests.get(url, headers={"User-Agent": settings.USER_AGENT})
|
||||
doc = BeautifulSoup(response.content, 'lxml')
|
||||
|
|
Loading…
Reference in New Issue