From 8b15438a9849ffdc9847b002392b3c6f1fd791d6 Mon Sep 17 00:00:00 2001 From: powe97 <116031952+powe97@users.noreply.github.com> Date: Wed, 6 Mar 2024 01:01:29 -0600 Subject: [PATCH] Actually use the retry version of the function... --- transfer_scraper/main.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/transfer_scraper/main.py b/transfer_scraper/main.py index 155c578..69a85a0 100644 --- a/transfer_scraper/main.py +++ b/transfer_scraper/main.py @@ -103,15 +103,17 @@ def scrape_page(page_num): driver.quit() print(f"Scraping page {page_num}, found {num_institutions} links", file=sys.stderr) - return [scrape_institution(i, page_num) for i in range(0, num_institutions)] + return [scrape_institution_safe(i, page_num) for i in range(0, num_institutions)] def scrape_institution_safe(index, page_num): - for i in range(1,15): + for i in range(1, 15): try: return scrape_institution(index, page_num) - except Exception: - print(f"\tAttempt {i} failed, trying again...") + except Exception as e: + print( + f"\tAttempt {i} failed due to {type(e).__name__}: {e}, trying again..." + ) sleep(40) raise Exception(f"Failed to scrape {index} after 15 attempts, aborting.")