phantop/quatalog-scraper
1
0
Fork 0
mirror of https://github.com/quatalog/quatalog.git synced 2025-11-29 07:47:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make failing actually fail the program

Browse source
This commit is contained in:
powe97 2024-03-06 02:35:18 -06:00
parent 92c3327b1a
commit 81ba2fdc80
No known key found for this signature in database
GPG key ID: 7D1663B10978D1BA
1 changed files with 6 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
transfer_scraper/main.py
View file

@ -98,11 +98,14 @@ def scrape_page(page_num):
jump_to_page(1, page_num, "gdvInstWithEQ", "lblInstWithEQPaginationInfo") jump_to_page(1, page_num, "gdvInstWithEQ", "lblInstWithEQPaginationInfo")
break break
except Exception as e: except Exception as e:
driver.quit()
print( print(
f"Attempt {i} failed to load page, retrying in 25 seconds...", f"Attempt {i} failed to load page, retrying in 25 seconds...",
file=sys.stderr, file=sys.stderr,
) )
sleep(25) sleep(25)
else:
raise Exception(f"Failed to load the main page after 15 attempts, aborting.")
num_institutions = len( num_institutions = len(
driver.find_elements( driver.find_elements(
@ -120,12 +123,14 @@ def scrape_institution_safe(index, page_num):
try: try:
return scrape_institution(index, page_num) return scrape_institution(index, page_num)
except Exception as e: except Exception as e:
driver.quit()
print( print(
f"\tAttempt {i} failed due to {type(e).__name__}: {e}, retrying in 25 seconds...", f"\tAttempt {i} failed due to {type(e).__name__}: {e}, retrying in 25 seconds...",
file=sys.stderr, file=sys.stderr,
) )
sleep(25) sleep(25)
raise Exception(f"Failed to scrape {index} after 15 attempts, aborting.") else:
raise Exception(f"Failed to scrape {index} after 15 attempts, aborting.")
# scrape_institution: Scrapes an institution by index. # scrape_institution: Scrapes an institution by index.