diff --git a/.github/workflows/scrape_page.yml b/.github/workflows/scrape_page.yml new file mode 100644 index 0000000..3c248b1 --- /dev/null +++ b/.github/workflows/scrape_page.yml @@ -0,0 +1,38 @@ +name: Scrape single page of transfer guide +run-name: Scrape single page of transfer guide +on: + workflow_call: + inputs: + page: + required: true + type: number + +jobs: + scrape-page: + runs-on: ubuntu-latest + steps: + - name: Checkout scraping repo + uses: actions/checkout@v4 + + - name: Set up python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + working-directory: transfer_scraper + run: | + python -m pip install --upgrade pip + pip install -r 'requirements.txt' + + - name: Run scraper + working-directory: transfer_scraper + run: | + python3 scrape_page.py ${{ inputs.page }} transfer_${{ inputs.page }}.json + + - name: Upload data to artifact + uses: actions/upload-artifact@v4 + with: + name: transfer-page-${{ matrix.page }} + path: transfer_scraper/transfer_${{ matrix.page }}.json