Compare commits
5 Commits
86da3caee7
...
f7c5c2461a
Author | SHA1 | Date |
---|---|---|
powe97 | f7c5c2461a | |
powe97 | 130dca8b89 | |
powe97 | 1403b42a5c | |
powe97 | 463a2aac03 | |
powe97 | d0f2f969d2 |
|
@ -0,0 +1,93 @@
|
|||
name: Update transfer guides
|
||||
run-name: Update transfer guides
|
||||
on:
|
||||
workflow_dispatch:
|
||||
repository_dispatch:
|
||||
types:
|
||||
generate-csv
|
||||
|
||||
jobs:
|
||||
by-courses-json:
|
||||
name: Generate by-courses JSON file
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Set up python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Checkout data repo
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: quatalog/data
|
||||
path: data
|
||||
token: ${{ secrets.PUSH_TOKEN }}
|
||||
|
||||
- name: Checkout scraping repo
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
path: scrapers
|
||||
|
||||
- name: Convert to by-courses format
|
||||
run: |
|
||||
python3 scrapers/transfer_scraper/convert_json.py data/transfer.json data/transfer_by_course.json
|
||||
|
||||
- name: Commit data
|
||||
working-directory: data
|
||||
run: |
|
||||
git config user.name "Quatalog Updater"
|
||||
git config user.email "github_actions@quatalog.com"
|
||||
git add transfer_*.json
|
||||
git commit -m "$(date)" || exit 0
|
||||
git push
|
||||
|
||||
generate-csvs:
|
||||
name: Generate transfer guides
|
||||
needs: by-courses-json
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Set up python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Checkout data repo
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: quatalog/data
|
||||
path: data
|
||||
token: ${{ secrets.PUSH_TOKEN }}
|
||||
|
||||
- name: Checkout transfer guides repo
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: quatalog/transfer_guides
|
||||
path: transfer_guides
|
||||
token: ${{ secrets.PUSH_TOKEN }}
|
||||
|
||||
- name: Checkout scraping repo
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
path: scrapers
|
||||
|
||||
- name: Generate CSVs
|
||||
run: |
|
||||
python3 scrapers/transfer_scraper/generate_csv.py data/transfer_by_course.json new_transfer_guides
|
||||
|
||||
- name: Remove stale entries
|
||||
working-directory: transfer_guides
|
||||
run: |
|
||||
git rm *
|
||||
|
||||
- name: Copy CSVs over
|
||||
run: |
|
||||
rsync -azvh new_transfer_guides/ transfer_guides
|
||||
|
||||
- name: Commit transfer guides
|
||||
working-directory: transfer_guides
|
||||
run: |
|
||||
git config user.name "Quatalog Updater"
|
||||
git config user.email "github_actions@quatalog.com"
|
||||
git add .
|
||||
git commit -m "$(date)" || exit 0
|
||||
git push
|
|
@ -59,6 +59,11 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
needs: scrape-page
|
||||
steps:
|
||||
- name: Set up python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Checkout data repo
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
|
@ -66,11 +71,6 @@ jobs:
|
|||
path: data
|
||||
token: ${{ secrets.PUSH_TOKEN }}
|
||||
|
||||
- name: Set up python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Checkout scraping repo
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
|
@ -87,15 +87,16 @@ jobs:
|
|||
run: |
|
||||
cat new-data/* | jq -s 'add | sort_by(.institution)' > data/transfer.json
|
||||
|
||||
- name: Convert to by-courses format
|
||||
run: |
|
||||
python scrapers/transfer_scraper/convert_json.py data/transfer.json data/transfer_by_course.json
|
||||
|
||||
- name: Commit data
|
||||
working-directory: data
|
||||
run: |
|
||||
git config user.name "Quatalog Updater"
|
||||
git config user.email "github_actions@quatalog.com"
|
||||
git add transfer*.json
|
||||
git add transfer.json
|
||||
git commit -m "$(date)" || exit 0
|
||||
git push
|
||||
|
||||
- name: Trigger CSV generator
|
||||
uses: peter-evans/repository-dispatch@v3
|
||||
with:
|
||||
event-type: generate-csv
|
||||
|
|
Loading…
Reference in New Issue