Create scrape_page.yml

This commit is contained in:
powe97 2024-03-16 20:46:25 -04:00 committed by GitHub
parent ea09d33ac2
commit ecfb176c46
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 38 additions and 0 deletions

38
.github/workflows/scrape_page.yml vendored Normal file
View File

@ -0,0 +1,38 @@
name: Scrape single page of transfer guide
run-name: Scrape single page of transfer guide
on:
workflow_call:
inputs:
page:
required: true
type: number
jobs:
scrape-page:
runs-on: ubuntu-latest
steps:
- name: Checkout scraping repo
uses: actions/checkout@v4
- name: Set up python
uses: actions/setup-python@v5
with:
python-version: '3.11'
cache: 'pip'
- name: Install dependencies
working-directory: transfer_scraper
run: |
python -m pip install --upgrade pip
pip install -r 'requirements.txt'
- name: Run scraper
working-directory: transfer_scraper
run: |
python3 scrape_page.py ${{ inputs.page }} transfer_${{ inputs.page }}.json
- name: Upload data to artifact
uses: actions/upload-artifact@v4
with:
name: transfer-page-${{ matrix.page }}
path: transfer_scraper/transfer_${{ matrix.page }}.json