phantop/quatalog-scraper
1
0
Fork 0
mirror of https://github.com/quatalog/quatalog.git synced 2024-11-23 07:13:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix global var fuckery

Browse source
This commit is contained in:
powe97 2024-02-29 21:21:39 -05:00
parent 4916feeb19
commit 12d844ca28
No known key found for this signature in database
GPG key ID: 7D1663B10978D1BA
1 changed files with 11 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
transfer_scraper/main.py
View file

@ -30,7 +30,9 @@ def normalize_class_name(input):
return "".join(text) return "".join(text)
def wait(driver, ec): def wait(ec):
global driver
WebDriverWait( WebDriverWait(
driver, 20, ignored_exceptions=[StaleElementReferenceException] driver, 20, ignored_exceptions=[StaleElementReferenceException]
).until(ec) ).until(ec)
@ -38,6 +40,8 @@ def wait(driver, ec):
def scrape_course_card(html_id, i, note): def scrape_course_card(html_id, i, note):
global driver
trs = ( trs = (
driver.find_element("id", html_id) driver.find_element("id", html_id)
.find_elements(By.CSS_SELECTOR, ".course-detail")[i] .find_elements(By.CSS_SELECTOR, ".course-detail")[i]
@ -101,8 +105,12 @@ def scrape_course_card(html_id, i, note):
def main(): def main():
global driver
if len(sys.argv) != 4: if len(sys.argv) != 4:
print(f"USAGE: python {sys.argv[0]} <transfer file> <state file> <timeout minutes>") print(
f"USAGE: python {sys.argv[0]} <transfer file> <state file> <timeout minutes>"
)
exit(1) exit(1)
transfer_json_path = sys.argv[1] transfer_json_path = sys.argv[1]
@ -119,6 +127,7 @@ def main():
if sys.argv[-1] != "gui": if sys.argv[-1] != "gui":
options.add_argument("--headless") options.add_argument("--headless")
options.set_preference("general.useragent.override", user_agent) options.set_preference("general.useragent.override", user_agent)
driver = webdriver.Firefox(options=options) driver = webdriver.Firefox(options=options)
driver.get( driver.get(
"https://tes.collegesource.com/publicview/TES_publicview01.aspx?rid=f080a477-bff8-46df-a5b2-25e9affdd4ed&aid=27b576bb-cd07-4e57-84d0-37475fde70ce" "https://tes.collegesource.com/publicview/TES_publicview01.aspx?rid=f080a477-bff8-46df-a5b2-25e9affdd4ed&aid=27b576bb-cd07-4e57-84d0-37475fde70ce"