diff --git a/.github/workflows/transfer.yml b/.github/workflows/transfer.yml index b7034b3..982b004 100644 --- a/.github/workflows/transfer.yml +++ b/.github/workflows/transfer.yml @@ -55,7 +55,7 @@ jobs: path: transfer_scraper/transfer_${{ matrix.page }}.json commit-data: - name: Combine and commit data + name: Combine/convert and commit data runs-on: ubuntu-latest needs: scrape-page steps: @@ -66,6 +66,16 @@ jobs: path: data token: ${{ secrets.PUSH_TOKEN }} + - name: Set up python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Checkout scraping repo + uses: actions/checkout@v4 + with: + path: scrapers + - name: Download partial JSONs uses: actions/download-artifact@v4 with: @@ -76,12 +86,16 @@ jobs: - name: Combine JSONs run: | cat new-data/* | jq -s 'add | sort_by(.institution)' > data/transfer.json + + - name: Convert to by-courses format + run: | + python scrapers/transfer_scraper/convert_json.py data/transfer.json data/transfer_by_course.json - name: Commit data working-directory: data run: | git config user.name "Quatalog Updater" git config user.email "github_actions@quatalog.com" - git add transfer.json + git add transfer*.json git commit -m "$(date)" || exit 0 git push