phantop
/
quatalog-scraper
mirror of https://github.com/quatalog/quatalog.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add termination

This commit is contained in:
powe97 2024-03-03 23:53:25 -05:00
parent 0b97a431e1
commit faf303ec27
No known key found for this signature in database
GPG Key ID: 7D1663B10978D1BA
1 changed files with 8 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
transfer_scraper/main.py
View File

@ -169,7 +169,10 @@ def main():
"https://tes.collegesource.com/publicview/TES_publicview01.aspx?rid=f080a477-bff8-46df-a5b2-25e9affdd4ed&aid=27b576bb-cd07-4e57-84d0-37475fde70ce"
)
print(f'Title is {driver.find_element(By.TAG_NAME, "title").get_attribute("innerText").strip()}', file=sys.stderr)
print(
f'Title is {driver.find_element(By.TAG_NAME, "title").get_attribute("innerText").strip()}',
file=sys.stderr,
)
num_pages = int(
driver.find_element("id", "lblInstWithEQPaginationInfo").text.split()[-1]
@ -189,6 +192,9 @@ def main():
json.dump(state, sys.stderr, indent=4)
print("", file=sys.stderr)
if state["inst_pg"] > num_pages:
raise Exception
try:
curr_inst_page = 1
while state["inst_pg"] <= num_pages:
@ -351,7 +357,7 @@ def main():
),
)
state["inst_idx"] = 0
state["inst_pg"] = (state["inst_pg"] % num_pages) + 1
state["inst_pg"] += 1
except (Exception, KeyboardInterrupt) as e:
print("Program hits exception and will save and terminate", file=sys.stderr)