diff --git a/.github/workflows/transfer.yml b/.github/workflows/transfer.yml index 80c6fbc..106d877 100644 --- a/.github/workflows/transfer.yml +++ b/.github/workflows/transfer.yml @@ -1,12 +1,11 @@ name: Scrape transfer and update file run-name: Scrape transfer and update file -env: - DEFAULT_TIMEOUT: 45 on: workflow_dispatch: jobs: - starter: + setup: + name: Get number of pages and set up scrape page jobs runs-on: ubuntu-latest steps: - name: Create matrix parameters @@ -19,10 +18,35 @@ jobs: outputs: matrix-params: ${{ steps.matrix-params.outputs.matrix-params }} matrix-job: + name: Scrape page runs-on: ubuntu-latest - needs: starter + needs: setup strategy: matrix: ${{ fromJson(needs.starter.outputs.matrix-params) }} steps: - - run: | - echo ${{ matrix.page }} + - name: Checkout scraping repo + uses: actions/checkout@v4 + + - name: Set up python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + working-directory: transfer_scraper + run: | + python -m pip install --upgrade pip + pip install -r 'requirements.txt' + + - name: Run scraper + working-directory: transfer_scraper + run: | + python3 main.py ${{ matrix.page }} transfer.json + + - name: Upload data to artifact + uses: actions/upload-artifact@v4 + with: + name: transfer-page-${{ matrix.page }} + path: transfer_scraper/transfer.json +