From de7360f1c75222d8105dbcee66f0f602de890492 Mon Sep 17 00:00:00 2001
From: powe97 <116031952+powe97@users.noreply.github.com>
Date: Sat, 16 Mar 2024 01:28:33 -0400
Subject: [PATCH] Convert JSON to by-course format

---
 .github/workflows/transfer.yml | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/transfer.yml b/.github/workflows/transfer.yml
index b7034b3..982b004 100644
--- a/.github/workflows/transfer.yml
+++ b/.github/workflows/transfer.yml
@@ -55,7 +55,7 @@ jobs:
           path: transfer_scraper/transfer_${{ matrix.page }}.json
 
   commit-data:
-    name: Combine and commit data
+    name: Combine/convert and commit data
     runs-on: ubuntu-latest
     needs: scrape-page
     steps:
@@ -66,6 +66,16 @@ jobs:
           path: data
           token: ${{ secrets.PUSH_TOKEN }}
 
+      - name: Set up python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Checkout scraping repo
+        uses: actions/checkout@v4
+        with:
+          path: scrapers
+
       - name: Download partial JSONs
         uses: actions/download-artifact@v4
         with:
@@ -76,12 +86,16 @@ jobs:
       - name: Combine JSONs
         run: |
           cat new-data/* | jq -s 'add | sort_by(.institution)' > data/transfer.json
+
+      - name: Convert to by-courses format
+        run: |
+          python scrapers/transfer_scraper/convert_json.py data/transfer.json data/transfer_by_course.json
       
       - name: Commit data
         working-directory: data
         run: |
           git config user.name "Quatalog Updater"
           git config user.email "github_actions@quatalog.com"
-          git add transfer.json
+          git add transfer*.json
           git commit -m "$(date)" || exit 0
           git push