mirror of
https://github.com/quatalog/quatalog.git
synced 2024-09-28 20:48:57 +00:00
Compare commits
5 commits
86da3caee7
...
f7c5c2461a
Author | SHA1 | Date | |
---|---|---|---|
f7c5c2461a | |||
130dca8b89 | |||
1403b42a5c | |||
463a2aac03 | |||
d0f2f969d2 |
93
.github/workflows/generate-csvs.yml
vendored
Normal file
93
.github/workflows/generate-csvs.yml
vendored
Normal file
|
@ -0,0 +1,93 @@
|
||||||
|
name: Update transfer guides
|
||||||
|
run-name: Update transfer guides
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
repository_dispatch:
|
||||||
|
types:
|
||||||
|
generate-csv
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
by-courses-json:
|
||||||
|
name: Generate by-courses JSON file
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Set up python
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: '3.11'
|
||||||
|
|
||||||
|
- name: Checkout data repo
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
repository: quatalog/data
|
||||||
|
path: data
|
||||||
|
token: ${{ secrets.PUSH_TOKEN }}
|
||||||
|
|
||||||
|
- name: Checkout scraping repo
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
path: scrapers
|
||||||
|
|
||||||
|
- name: Convert to by-courses format
|
||||||
|
run: |
|
||||||
|
python3 scrapers/transfer_scraper/convert_json.py data/transfer.json data/transfer_by_course.json
|
||||||
|
|
||||||
|
- name: Commit data
|
||||||
|
working-directory: data
|
||||||
|
run: |
|
||||||
|
git config user.name "Quatalog Updater"
|
||||||
|
git config user.email "github_actions@quatalog.com"
|
||||||
|
git add transfer_*.json
|
||||||
|
git commit -m "$(date)" || exit 0
|
||||||
|
git push
|
||||||
|
|
||||||
|
generate-csvs:
|
||||||
|
name: Generate transfer guides
|
||||||
|
needs: by-courses-json
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Set up python
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: '3.11'
|
||||||
|
|
||||||
|
- name: Checkout data repo
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
repository: quatalog/data
|
||||||
|
path: data
|
||||||
|
token: ${{ secrets.PUSH_TOKEN }}
|
||||||
|
|
||||||
|
- name: Checkout transfer guides repo
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
repository: quatalog/transfer_guides
|
||||||
|
path: transfer_guides
|
||||||
|
token: ${{ secrets.PUSH_TOKEN }}
|
||||||
|
|
||||||
|
- name: Checkout scraping repo
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
path: scrapers
|
||||||
|
|
||||||
|
- name: Generate CSVs
|
||||||
|
run: |
|
||||||
|
python3 scrapers/transfer_scraper/generate_csv.py data/transfer_by_course.json new_transfer_guides
|
||||||
|
|
||||||
|
- name: Remove stale entries
|
||||||
|
working-directory: transfer_guides
|
||||||
|
run: |
|
||||||
|
git rm *
|
||||||
|
|
||||||
|
- name: Copy CSVs over
|
||||||
|
run: |
|
||||||
|
rsync -azvh new_transfer_guides/ transfer_guides
|
||||||
|
|
||||||
|
- name: Commit transfer guides
|
||||||
|
working-directory: transfer_guides
|
||||||
|
run: |
|
||||||
|
git config user.name "Quatalog Updater"
|
||||||
|
git config user.email "github_actions@quatalog.com"
|
||||||
|
git add .
|
||||||
|
git commit -m "$(date)" || exit 0
|
||||||
|
git push
|
21
.github/workflows/transfer.yml
vendored
21
.github/workflows/transfer.yml
vendored
|
@ -59,6 +59,11 @@ jobs:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: scrape-page
|
needs: scrape-page
|
||||||
steps:
|
steps:
|
||||||
|
- name: Set up python
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: '3.11'
|
||||||
|
|
||||||
- name: Checkout data repo
|
- name: Checkout data repo
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
|
@ -66,11 +71,6 @@ jobs:
|
||||||
path: data
|
path: data
|
||||||
token: ${{ secrets.PUSH_TOKEN }}
|
token: ${{ secrets.PUSH_TOKEN }}
|
||||||
|
|
||||||
- name: Set up python
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: '3.11'
|
|
||||||
|
|
||||||
- name: Checkout scraping repo
|
- name: Checkout scraping repo
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
|
@ -87,15 +87,16 @@ jobs:
|
||||||
run: |
|
run: |
|
||||||
cat new-data/* | jq -s 'add | sort_by(.institution)' > data/transfer.json
|
cat new-data/* | jq -s 'add | sort_by(.institution)' > data/transfer.json
|
||||||
|
|
||||||
- name: Convert to by-courses format
|
|
||||||
run: |
|
|
||||||
python scrapers/transfer_scraper/convert_json.py data/transfer.json data/transfer_by_course.json
|
|
||||||
|
|
||||||
- name: Commit data
|
- name: Commit data
|
||||||
working-directory: data
|
working-directory: data
|
||||||
run: |
|
run: |
|
||||||
git config user.name "Quatalog Updater"
|
git config user.name "Quatalog Updater"
|
||||||
git config user.email "github_actions@quatalog.com"
|
git config user.email "github_actions@quatalog.com"
|
||||||
git add transfer*.json
|
git add transfer.json
|
||||||
git commit -m "$(date)" || exit 0
|
git commit -m "$(date)" || exit 0
|
||||||
git push
|
git push
|
||||||
|
|
||||||
|
- name: Trigger CSV generator
|
||||||
|
uses: peter-evans/repository-dispatch@v3
|
||||||
|
with:
|
||||||
|
event-type: generate-csv
|
||||||
|
|
Loading…
Reference in a new issue