diff --git a/.github/workflows/transfer.yml b/.github/workflows/transfer.yml index 982b004..c55941f 100644 --- a/.github/workflows/transfer.yml +++ b/.github/workflows/transfer.yml @@ -59,6 +59,11 @@ jobs: runs-on: ubuntu-latest needs: scrape-page steps: + - name: Set up python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Checkout data repo uses: actions/checkout@v4 with: @@ -66,11 +71,6 @@ jobs: path: data token: ${{ secrets.PUSH_TOKEN }} - - name: Set up python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Checkout scraping repo uses: actions/checkout@v4 with: @@ -99,3 +99,51 @@ jobs: git add transfer*.json git commit -m "$(date)" || exit 0 git push + + generate-csv: + name: Generate CSVs + runs-on: ubuntu-latest + needs: commit-data + steps: + - name: Set up python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Checkout data repo + uses: actions/checkout@v4 + with: + repository: quatalog/data + path: data + + - name: Checkout scrapers repo + uses: actions/checkout@v4 + with: + path: scrapers + + - name: Checkout transfer guides repo + uses: actions/checkout@v4 + with: + repository: quatalog/transfer_guides + path: transfer_guides + token: ${{ secrets.PUSH_TOKEN }} + + - name: Generate CSVs + run: | + python scrapers/generate_csv.py data/transfer_by_course.json new_transfer_guides + + - name: Copy data to repo and remove stale files + run: | + cd transfer_guides + git rm * + cd .. + rsync -azvh new_transfer_guides/ transfer_guides + + - name: Commit CSVs + working-directory: transfer_guides + run: | + git config user.name "Quatalog Updater" + git config user.email "github_actions@quatalog.com" + git add . + git commit -m "$(date)" || exit 0 + git push