EbookFoundation
/
regluit
mirror of https://github.com/EbookFoundation/regluit.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add ua setting to generic 1 harvest

pull/94/head
eric 2020-08-11 18:19:10 -04:00
parent 6341b6a1b7
commit 457c66e098
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
core/loaders/harvest.py
View File

@ -178,8 +178,8 @@ def make_harvested_ebook(content, ebook, format, filesize=0):
return new_ebf, 1 return new_ebf, 1
def harvest_one_generic(ebook, selector): def harvest_one_generic(ebook, selector, user_agent=settings.USER_AGENT):
doc = get_soup(ebook.url) doc = get_soup(ebook.url, user_agent=user_agent)
if doc: if doc:
try: try:
base = doc.find('base')['href'] base = doc.find('base')['href']
@ -188,7 +188,7 @@ def harvest_one_generic(ebook, selector):
obj = selector(doc) obj = selector(doc)
if obj: if obj:
dl_url = urljoin(base, obj['href']) dl_url = urljoin(base, obj['href'])
harvest = make_dl_ebook(dl_url, ebook) harvest = make_dl_ebook(dl_url, ebook, user_agent=user_agent)
if not harvest[0]: if not harvest[0]:
logger.warning('couldn\'t harvest %s', dl_url) logger.warning('couldn\'t harvest %s', dl_url)
return harvest return harvest