phantop
/
quatalog-scraper
mirror of https://github.com/quatalog/quatalog.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix issue where only 1 page per school would get scraped properly

This commit is contained in:
powe97 2024-03-01 18:16:58 -05:00
parent 5ea6816c90
commit baa74b8ee6
No known key found for this signature in database
GPG Key ID: 7D1663B10978D1BA
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
transfer_scraper/main.py
View File

@ -126,6 +126,10 @@ def main():
user_agent = UserAgent().random
options.set_preference("general.useragent.override", user_agent)
# options.set_preference("network.proxy.socks", )
# options.set_preference("network.proxy.socks_port", )
# options.set_preference("network.proxy.socks_remote_dns", True)
# options.set_preference("network.proxy.type", 1)
print(f"Using randomized user agent {user_agent}", file=sys.stderr)
driver = webdriver.Firefox(options=options)
@ -211,7 +215,7 @@ def main():
try:
course_pages_len = int(
driver.find_element(
"id", "lblInstWithEQPaginationInfo"
"id", "lblCourseEQPaginationInfo"
).text.split()[-1]
)
except NoSuchElementException: