phantop/quatalog-scraper
1
0
Fork 0
mirror of https://github.com/quatalog/quatalog.git synced 2024-11-15 19:42:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix issue where only 1 page per school would get scraped properly

Browse source
This commit is contained in:
powe97 2024-03-01 18:16:58 -05:00
parent 5ea6816c90
commit baa74b8ee6
No known key found for this signature in database
GPG key ID: 7D1663B10978D1BA
1 changed files with 5 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
transfer_scraper/main.py
View file

@ -126,6 +126,10 @@ def main():
user_agent = UserAgent().random user_agent = UserAgent().random
options.set_preference("general.useragent.override", user_agent) options.set_preference("general.useragent.override", user_agent)
# options.set_preference("network.proxy.socks", )
# options.set_preference("network.proxy.socks_port", )
# options.set_preference("network.proxy.socks_remote_dns", True)
# options.set_preference("network.proxy.type", 1)
print(f"Using randomized user agent {user_agent}", file=sys.stderr) print(f"Using randomized user agent {user_agent}", file=sys.stderr)
driver = webdriver.Firefox(options=options) driver = webdriver.Firefox(options=options)
@ -211,7 +215,7 @@ def main():
try: try:
course_pages_len = int( course_pages_len = int(
driver.find_element( driver.find_element(
"id", "lblInstWithEQPaginationInfo" "id", "lblCourseEQPaginationInfo"
).text.split()[-1] ).text.split()[-1]
) )
except NoSuchElementException: except NoSuchElementException: