phantop
/
quatalog-scraper
mirror of https://github.com/quatalog/quatalog.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix debug prints

This commit is contained in:
powe97 2024-03-06 02:09:42 -06:00
parent 6a1395c054
commit 92c3327b1a
No known key found for this signature in database
GPG Key ID: 7D1663B10978D1BA
1 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
transfer_scraper/main.py
View File

@ -98,7 +98,10 @@ def scrape_page(page_num):
jump_to_page(1, page_num, "gdvInstWithEQ", "lblInstWithEQPaginationInfo")
break
except Exception as e:
print("Attempt {i} failed to load page, retrying in 25 seconds...")
print(
f"Attempt {i} failed to load page, retrying in 25 seconds...",
file=sys.stderr,
)
sleep(25)
num_institutions = len(
@ -118,7 +121,8 @@ def scrape_institution_safe(index, page_num):
return scrape_institution(index, page_num)
except Exception as e:
print(
f"\tAttempt {i} failed due to {type(e).__name__}: {e}, retrying in 25 seconds..."
f"\tAttempt {i} failed due to {type(e).__name__}: {e}, retrying in 25 seconds...",
file=sys.stderr,
)
sleep(25)
raise Exception(f"Failed to scrape {index} after 15 attempts, aborting.")