diff --git a/transfer_scraper/main.py b/transfer_scraper/main.py index 770a3da..28c20ae 100644 --- a/transfer_scraper/main.py +++ b/transfer_scraper/main.py @@ -30,7 +30,7 @@ def normalize_class_name(input): return "".join(text) -def wait(ec): +def wait(driver, ec): WebDriverWait( driver, 20, ignored_exceptions=[StaleElementReferenceException] ).until(ec) @@ -171,7 +171,7 @@ def main(): jumpable_pages[max(jumpable_pages)].click() curr_page = max(jumpable_pages) - wait(EC.staleness_of(page)) + wait(driver, EC.staleness_of(page)) sleep(random.uniform(3, 6)) page = driver.find_element("id", f"gdvInstWithEQ") @@ -196,7 +196,7 @@ def main(): us_state = fields[1].text.strip() institution_link.click() - wait(EC.staleness_of(institution_link)) + wait(driver, EC.staleness_of(institution_link)) try: course_pages_len = int( @@ -230,9 +230,10 @@ def main(): try: wait( + driver, EC.element_to_be_clickable( (By.CSS_SELECTOR, ".modal-header button") - ) + ), ) transfer = [ @@ -309,9 +310,10 @@ def main(): driver.find_element("id", "btnSwitchView").click() wait( + driver, EC.text_to_be_present_in_element( ("id", "lblInstWithEQPaginationInfo"), str(state["inst_pg"]) - ) + ), ) state["inst_idx"] = 0 state["inst_pg"] = (state["inst_pg"] % num_pages) + 1