phantop/quatalog-scraper
1
0
Fork 0
mirror of https://github.com/quatalog/quatalog.git synced 2025-12-01 00:27:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix scraper

Browse source
This commit is contained in:
powe97 2024-02-29 21:02:38 -05:00
parent f216c45748
commit b304e9f8d2
No known key found for this signature in database
GPG key ID: 7D1663B10978D1BA
1 changed files with 7 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
transfer_scraper/main.py
View file

@ -30,7 +30,7 @@ def normalize_class_name(input):
return "".join(text)
def wait(ec):
def wait(driver, ec):
WebDriverWait(
driver, 20, ignored_exceptions=[StaleElementReferenceException]
).until(ec)
@ -171,7 +171,7 @@ def main():
jumpable_pages[max(jumpable_pages)].click()
curr_page = max(jumpable_pages)
wait(EC.staleness_of(page))
wait(driver, EC.staleness_of(page))
sleep(random.uniform(3, 6))
page = driver.find_element("id", f"gdvInstWithEQ")
@ -196,7 +196,7 @@ def main():
us_state = fields[1].text.strip()
institution_link.click()
wait(EC.staleness_of(institution_link))
wait(driver, EC.staleness_of(institution_link))
try:
course_pages_len = int(
@ -230,9 +230,10 @@ def main():
try:
wait(
driver,
EC.element_to_be_clickable(
(By.CSS_SELECTOR, ".modal-header button")
)
),
)
transfer = [
@ -309,9 +310,10 @@ def main():
driver.find_element("id", "btnSwitchView").click()
wait(
driver,
EC.text_to_be_present_in_element(
("id", "lblInstWithEQPaginationInfo"), str(state["inst_pg"])
)
),
)
state["inst_idx"] = 0
state["inst_pg"] = (state["inst_pg"] % num_pages) + 1