From faf303ec27ec90edb27eec6de19029f384da38ff Mon Sep 17 00:00:00 2001 From: powe97 <116031952+powe97@users.noreply.github.com> Date: Sun, 3 Mar 2024 23:53:25 -0500 Subject: [PATCH] Add termination --- transfer_scraper/main.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/transfer_scraper/main.py b/transfer_scraper/main.py index 1c341f7..72e0b2c 100644 --- a/transfer_scraper/main.py +++ b/transfer_scraper/main.py @@ -169,7 +169,10 @@ def main(): "https://tes.collegesource.com/publicview/TES_publicview01.aspx?rid=f080a477-bff8-46df-a5b2-25e9affdd4ed&aid=27b576bb-cd07-4e57-84d0-37475fde70ce" ) - print(f'Title is {driver.find_element(By.TAG_NAME, "title").get_attribute("innerText").strip()}', file=sys.stderr) + print( + f'Title is {driver.find_element(By.TAG_NAME, "title").get_attribute("innerText").strip()}', + file=sys.stderr, + ) num_pages = int( driver.find_element("id", "lblInstWithEQPaginationInfo").text.split()[-1] @@ -189,6 +192,9 @@ def main(): json.dump(state, sys.stderr, indent=4) print("", file=sys.stderr) + if state["inst_pg"] > num_pages: + raise Exception + try: curr_inst_page = 1 while state["inst_pg"] <= num_pages: @@ -351,7 +357,7 @@ def main(): ), ) state["inst_idx"] = 0 - state["inst_pg"] = (state["inst_pg"] % num_pages) + 1 + state["inst_pg"] += 1 except (Exception, KeyboardInterrupt) as e: print("Program hits exception and will save and terminate", file=sys.stderr)