phantop/quatalog-scraper
1
0
Fork 0
mirror of https://github.com/quatalog/quatalog.git synced 2025-02-10 20:52:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Re-add catalog name scraping

Browse source
This commit is contained in:
powe97 2024-03-13 23:49:57 -04:00
parent f1a47dca48
commit 517952f977
No known key found for this signature in database
GPG key ID: 7D1663B10978D1BA
1 changed files with 10 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
transfer_scraper/main.py
View file

@ -94,7 +94,10 @@ def scrape_page(page_num):
try:
driver = webdriver.Firefox(options=options)
driver.get("https://ipinfo.io/ip")
print(f"Trying with IP {driver.page_source}", file=sys.stderr)
print(
f'Trying with IP {driver.find_element(By.TAG_NAME, "body").text}',
file=sys.stderr,
)
driver.get(
"https://tes.collegesource.com/publicview/TES_publicview01.aspx?rid=f080a477-bff8-46df-a5b2-25e9affdd4ed&aid=27b576bb-cd07-4e57-84d0-37475fde70ce"
)
@ -184,7 +187,7 @@ def scrape_institution(index, page_num):
"institution": inst_name,
"city": inst_city,
"state": inst_state,
"courses": [],
"transfers": [],
}
# Open list
@ -215,7 +218,7 @@ def scrape_institution(index, page_num):
"institution": inst_name,
"city": inst_city,
"state": inst_state,
"courses": transfer_courses,
"transfers": transfer_courses,
}
@ -234,7 +237,10 @@ def parse_course_td(td, include_credits):
for x in td_text[: len(td_text) - 3]
]
return [parse_one_course(x, include_credits) for x in courses_info]
return {
"catalog": td.find_element(By.TAG_NAME, "span").text.strip(),
"courses": [parse_one_course(x, include_credits) for x in courses_info],
}
def parse_one_course(course_info, include_credits):