From 81ba2fdc80fbf8cba993e640f3d9d8c9ed48ee84 Mon Sep 17 00:00:00 2001
From: powe97 <116031952+powe97@users.noreply.github.com>
Date: Wed, 6 Mar 2024 02:35:18 -0600
Subject: [PATCH] Make failing actually fail the program
---
transfer_scraper/main.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/transfer_scraper/main.py b/transfer_scraper/main.py
index 85bb087..e97c4f5 100644
--- a/transfer_scraper/main.py
+++ b/transfer_scraper/main.py
@@ -98,11 +98,14 @@ def scrape_page(page_num):
jump_to_page(1, page_num, "gdvInstWithEQ", "lblInstWithEQPaginationInfo")
break
except Exception as e:
+ driver.quit()
print(
f"Attempt {i} failed to load page, retrying in 25 seconds...",
file=sys.stderr,
)
sleep(25)
+ else:
+ raise Exception(f"Failed to load the main page after 15 attempts, aborting.")
num_institutions = len(
driver.find_elements(
@@ -120,12 +123,14 @@ def scrape_institution_safe(index, page_num):
try:
return scrape_institution(index, page_num)
except Exception as e:
+ driver.quit()
print(
f"\tAttempt {i} failed due to {type(e).__name__}: {e}, retrying in 25 seconds...",
file=sys.stderr,
)
sleep(25)
- raise Exception(f"Failed to scrape {index} after 15 attempts, aborting.")
+ else:
+ raise Exception(f"Failed to scrape {index} after 15 attempts, aborting.")
# scrape_institution: Scrapes an institution by index.