phantop/quatalog-scraper
1
0
Fork 0
mirror of https://github.com/quatalog/quatalog.git synced 2025-11-30 08:07:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Log IP

Browse source
This commit is contained in:
powe97 2024-03-13 22:18:13 -04:00
parent b017436be9
commit af25410c5d
No known key found for this signature in database
GPG key ID: 7D1663B10978D1BA
1 changed files with 5 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
transfer_scraper/main.py
View file

@ -93,6 +93,8 @@ def scrape_page(page_num):
for i in range(1, 4): for i in range(1, 4):
try: try:
driver = webdriver.Firefox(options=options) driver = webdriver.Firefox(options=options)
driver.get("https://ipinfo.io/ip")
print(f"Trying with IP {driver.page_source}")
driver.get( driver.get(
"https://tes.collegesource.com/publicview/TES_publicview01.aspx?rid=f080a477-bff8-46df-a5b2-25e9affdd4ed&aid=27b576bb-cd07-4e57-84d0-37475fde70ce" "https://tes.collegesource.com/publicview/TES_publicview01.aspx?rid=f080a477-bff8-46df-a5b2-25e9affdd4ed&aid=27b576bb-cd07-4e57-84d0-37475fde70ce"
) )
@ -102,13 +104,14 @@ def scrape_page(page_num):
break break
except Exception as e: except Exception as e:
driver.quit() driver.quit()
print( print(
f"Attempt {i} failed due to {type(e).__name__}, retrying in 25 seconds...", f"Attempt {i} failed due to {type(e).__name__}, retrying in 25 seconds...",
file=sys.stderr, file=sys.stderr,
) )
sleep(25) sleep(25)
else: else:
raise Exception(f"Failed to load the main page after 15 attempts, aborting.") raise Exception(f"Failed to load the main page after 4 attempts, aborting.")
num_institutions = len( num_institutions = len(
driver.find_elements( driver.find_elements(
@ -133,7 +136,7 @@ def scrape_institution_safe(index, page_num):
) )
sleep(25) sleep(25)
else: else:
raise Exception(f"Failed to scrape {index} after 15 attempts, aborting.") raise Exception(f"Failed to scrape {index} after 4 attempts, aborting.")
# scrape_institution: Scrapes an institution by index. # scrape_institution: Scrapes an institution by index.