phantop/quatalog-scraper
1
0
Fork 0
mirror of https://github.com/quatalog/quatalog.git synced 2025-11-30 16:17:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Actually use the retry version of the function...

Browse source
This commit is contained in:
powe97 2024-03-06 01:01:29 -06:00
parent 0007bde18a
commit 8b15438a98
No known key found for this signature in database
GPG key ID: 7D1663B10978D1BA
1 changed files with 6 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
transfer_scraper/main.py
View file

@ -103,15 +103,17 @@ def scrape_page(page_num):
driver.quit()
print(f"Scraping page {page_num}, found {num_institutions} links", file=sys.stderr)
return [scrape_institution(i, page_num) for i in range(0, num_institutions)]
return [scrape_institution_safe(i, page_num) for i in range(0, num_institutions)]
def scrape_institution_safe(index, page_num):
for i in range(1,15):
for i in range(1, 15):
try:
return scrape_institution(index, page_num)
except Exception:
print(f"\tAttempt {i} failed, trying again...")
except Exception as e:
print(
f"\tAttempt {i} failed due to {type(e).__name__}: {e}, trying again..."
)
sleep(40)
raise Exception(f"Failed to scrape {index} after 15 attempts, aborting.")