phantop/quatalog-scraper
1
0
Fork 0
mirror of https://github.com/quatalog/quatalog.git synced 2024-12-27 07:06:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Short circuit on IP ban

Browse source
This commit is contained in:
powe97 2024-03-16 21:07:39 -04:00
parent ecfb176c46
commit bdc6b2bcbc
No known key found for this signature in database
GPG key ID: 7D1663B10978D1BA
1 changed files with 10 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
transfer_scraper/scrape_page.py
View file

@ -16,6 +16,10 @@ from selenium.common.exceptions import (
)
class IPBanException(Exception):
pass
# Fix course titles accounting for Roman numerals up to X
def normalize_title(input):
s = " ".join(input.split())
@ -103,8 +107,14 @@ def scrape_page(page_num):
)
wait(EC.visibility_of_element_located((By.TAG_NAME, "body")))
print(f'Title: "{driver.title}"', file=sys.stderr)
if driver.title == "403 Forbidden":
raise IPBanException
jump_to_page(1, page_num, "gdvInstWithEQ", "lblInstWithEQPaginationInfo")
break
except IPBanException as e:
driver.quit()
print(f"We are IP-banned, exiting now", file=sys.stderr)
raise e
except Exception as e:
driver.quit()