mirror of
https://github.com/quatalog/quatalog.git
synced 2024-09-28 20:48:57 +00:00
Convert JSON to by-course format
This commit is contained in:
parent
ba3e5c77d1
commit
de7360f1c7
18
.github/workflows/transfer.yml
vendored
18
.github/workflows/transfer.yml
vendored
|
@ -55,7 +55,7 @@ jobs:
|
||||||
path: transfer_scraper/transfer_${{ matrix.page }}.json
|
path: transfer_scraper/transfer_${{ matrix.page }}.json
|
||||||
|
|
||||||
commit-data:
|
commit-data:
|
||||||
name: Combine and commit data
|
name: Combine/convert and commit data
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: scrape-page
|
needs: scrape-page
|
||||||
steps:
|
steps:
|
||||||
|
@ -66,6 +66,16 @@ jobs:
|
||||||
path: data
|
path: data
|
||||||
token: ${{ secrets.PUSH_TOKEN }}
|
token: ${{ secrets.PUSH_TOKEN }}
|
||||||
|
|
||||||
|
- name: Set up python
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: '3.11'
|
||||||
|
|
||||||
|
- name: Checkout scraping repo
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
path: scrapers
|
||||||
|
|
||||||
- name: Download partial JSONs
|
- name: Download partial JSONs
|
||||||
uses: actions/download-artifact@v4
|
uses: actions/download-artifact@v4
|
||||||
with:
|
with:
|
||||||
|
@ -77,11 +87,15 @@ jobs:
|
||||||
run: |
|
run: |
|
||||||
cat new-data/* | jq -s 'add | sort_by(.institution)' > data/transfer.json
|
cat new-data/* | jq -s 'add | sort_by(.institution)' > data/transfer.json
|
||||||
|
|
||||||
|
- name: Convert to by-courses format
|
||||||
|
run: |
|
||||||
|
python scrapers/transfer_scraper/convert_json.py data/transfer.json data/transfer_by_course.json
|
||||||
|
|
||||||
- name: Commit data
|
- name: Commit data
|
||||||
working-directory: data
|
working-directory: data
|
||||||
run: |
|
run: |
|
||||||
git config user.name "Quatalog Updater"
|
git config user.name "Quatalog Updater"
|
||||||
git config user.email "github_actions@quatalog.com"
|
git config user.email "github_actions@quatalog.com"
|
||||||
git add transfer.json
|
git add transfer*.json
|
||||||
git commit -m "$(date)" || exit 0
|
git commit -m "$(date)" || exit 0
|
||||||
git push
|
git push
|
||||||
|
|
Loading…
Reference in a new issue