phantop/quatalog-scraper
1
0
Fork 0
mirror of https://github.com/quatalog/quatalog.git synced 2025-12-01 16:49:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix scraper

Browse source
This commit is contained in:
powe97 2024-02-29 21:02:38 -05:00
parent f216c45748
commit b304e9f8d2
No known key found for this signature in database
GPG key ID: 7D1663B10978D1BA
1 changed files with 7 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
transfer_scraper/main.py
View file

@ -30,7 +30,7 @@ def normalize_class_name(input):
return "".join(text) return "".join(text)
def wait(ec): def wait(driver, ec):
WebDriverWait( WebDriverWait(
driver, 20, ignored_exceptions=[StaleElementReferenceException] driver, 20, ignored_exceptions=[StaleElementReferenceException]
).until(ec) ).until(ec)
@ -171,7 +171,7 @@ def main():
jumpable_pages[max(jumpable_pages)].click() jumpable_pages[max(jumpable_pages)].click()
curr_page = max(jumpable_pages) curr_page = max(jumpable_pages)
wait(EC.staleness_of(page)) wait(driver, EC.staleness_of(page))
sleep(random.uniform(3, 6)) sleep(random.uniform(3, 6))
page = driver.find_element("id", f"gdvInstWithEQ") page = driver.find_element("id", f"gdvInstWithEQ")
@ -196,7 +196,7 @@ def main():
us_state = fields[1].text.strip() us_state = fields[1].text.strip()
institution_link.click() institution_link.click()
wait(EC.staleness_of(institution_link)) wait(driver, EC.staleness_of(institution_link))
try: try:
course_pages_len = int( course_pages_len = int(
@ -230,9 +230,10 @@ def main():
try: try:
wait( wait(
driver,
EC.element_to_be_clickable( EC.element_to_be_clickable(
(By.CSS_SELECTOR, ".modal-header button") (By.CSS_SELECTOR, ".modal-header button")
) ),
) )
transfer = [ transfer = [
@ -309,9 +310,10 @@ def main():
driver.find_element("id", "btnSwitchView").click() driver.find_element("id", "btnSwitchView").click()
wait( wait(
driver,
EC.text_to_be_present_in_element( EC.text_to_be_present_in_element(
("id", "lblInstWithEQPaginationInfo"), str(state["inst_pg"]) ("id", "lblInstWithEQPaginationInfo"), str(state["inst_pg"])
) ),
) )
state["inst_idx"] = 0 state["inst_idx"] = 0
state["inst_pg"] = (state["inst_pg"] % num_pages) + 1 state["inst_pg"] = (state["inst_pg"] % num_pages) + 1