.github/workflows/transfer.yml

@@ -75,7 +75,7 @@ jobs:
 
       - name: Combine JSONs
         run: |
-          cat new-data/* | jq -s 'add | sort_by(.institution)' > data/transfer.json
+          cat new-data/* | jq -s 'add' > data/transfer.json
       
       - name: Commit data
         working-directory: data

transfer_scraper/main.py

@@ -29,7 +29,7 @@ def wait(ec):
     global driver
 
     WebDriverWait(
-        driver, 35, ignored_exceptions=[StaleElementReferenceException]
+        driver, 40, ignored_exceptions=[StaleElementReferenceException]
     ).until(ec)
     sleep(random.uniform(400, 1900) / 1000)
 
@@ -44,7 +44,6 @@ def wait(ec):
 def jump_to_page(curr_page, to_page, postback_type, pagination_type):
     global driver
 
-    wait(EC.visibility_of_element_located((By.ID, postback_type)))
     page = driver.find_element(By.ID, postback_type)
     try:
         num_pages = int(driver.find_element(By.ID, pagination_type).text.split()[-1])
@@ -101,7 +100,7 @@ def scrape_page(page_num):
         except Exception as e:
             driver.quit()
             print(
-                f"Attempt {i} failed due to {type(e).__name__}, retrying in 25 seconds...",
+                f"Attempt {i} failed to load page, retrying in 25 seconds...",
                 file=sys.stderr,
             )
             sleep(25)
@@ -126,7 +125,7 @@ def scrape_institution_safe(index, page_num):
         except Exception as e:
             driver.quit()
             print(
-                f"\tAttempt {i} failed due to {type(e).__name__}, retrying in 25 seconds...",
+                f"\tAttempt {i} failed due to {type(e).__name__}: {e}, retrying in 25 seconds...",
                 file=sys.stderr,
             )
             sleep(25)