mirror of
https://github.com/quatalog/quatalog.git
synced 2025-11-30 16:17:25 +00:00
Convert to use reusable workflow
This commit is contained in:
parent
f7c5c2461a
commit
ea09d33ac2
30
.github/workflows/transfer.yml
vendored
30
.github/workflows/transfer.yml
vendored
|
|
@ -22,37 +22,13 @@ jobs:
|
||||||
|
|
||||||
scrape-page:
|
scrape-page:
|
||||||
name: Scrape page
|
name: Scrape page
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: setup
|
needs: setup
|
||||||
|
uses: ./.github/workflows/scrape_page.yml
|
||||||
strategy:
|
strategy:
|
||||||
matrix: ${{ fromJson(needs.setup.outputs.matrix-params) }}
|
matrix: ${{ fromJson(needs.setup.outputs.matrix-params) }}
|
||||||
fail-fast: true
|
fail-fast: true
|
||||||
steps:
|
with:
|
||||||
- name: Checkout scraping repo
|
page: ${{ matrix.page }}
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Set up python
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: '3.11'
|
|
||||||
cache: 'pip'
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
working-directory: transfer_scraper
|
|
||||||
run: |
|
|
||||||
python -m pip install --upgrade pip
|
|
||||||
pip install -r 'requirements.txt'
|
|
||||||
|
|
||||||
- name: Run scraper
|
|
||||||
working-directory: transfer_scraper
|
|
||||||
run: |
|
|
||||||
python3 scrape_page.py ${{ matrix.page }} transfer_${{ matrix.page }}.json
|
|
||||||
|
|
||||||
- name: Upload data to artifact
|
|
||||||
uses: actions/upload-artifact@v4
|
|
||||||
with:
|
|
||||||
name: transfer-page-${{ matrix.page }}
|
|
||||||
path: transfer_scraper/transfer_${{ matrix.page }}.json
|
|
||||||
|
|
||||||
commit-data:
|
commit-data:
|
||||||
name: Combine/convert and commit data
|
name: Combine/convert and commit data
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue