phantop/quatalog-scraper
1
0
Fork 0
mirror of https://github.com/quatalog/quatalog.git synced 2024-11-08 16:14:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add retrying

Browse source
This commit is contained in:
powe97 2024-03-05 22:54:42 -05:00
parent a0b9081f8f
commit c98b928125
No known key found for this signature in database
GPG key ID: 7D1663B10978D1BA
1 changed files with 10 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
transfer_scraper/main.py
View file

@ -106,6 +106,16 @@ def scrape_page(page_num):
return [scrape_institution(i, page_num) for i in range(0, num_institutions)]
def scrape_institution_safe(index, page_num):
for i in range(1,15):
try:
return scrape_institution(index, page_num)
except Exception:
print(f"\tAttempt {i} failed, trying again...")
sleep(40)
raise Exception(f"Failed to scrape {index} after 15 attempts, aborting.")
# scrape_institution: Scrapes an institution by index.
#
# index: the 0-indexed index of the instituion to scrape on the page we are on.