Set up matrix jobs

.github/workflows/transfer.yml

@@ -1,12 +1,11 @@
 name: Scrape transfer and update file
 run-name: Scrape transfer and update file
-env:
-  DEFAULT_TIMEOUT: 45
 on:
   workflow_dispatch:
 
 jobs:
-  starter:
+  setup:
+    name: Get number of pages and set up scrape page jobs
     runs-on: ubuntu-latest
     steps:
     - name: Create matrix parameters
@@ -19,10 +18,35 @@ jobs:
     outputs:
       matrix-params: ${{ steps.matrix-params.outputs.matrix-params }}
   matrix-job:
+    name: Scrape page
     runs-on: ubuntu-latest
-    needs: starter
+    needs: setup
     strategy:
       matrix: ${{ fromJson(needs.starter.outputs.matrix-params) }}
     steps:
-      - run: |
-          echo ${{ matrix.page }}
+      - name: Checkout scraping repo
+        uses: actions/checkout@v4
+
+      - name: Set up python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+
+      - name: Install dependencies
+        working-directory: transfer_scraper
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r 'requirements.txt'
+
+      - name: Run scraper
+        working-directory: transfer_scraper
+        run: |
+          python3 main.py ${{ matrix.page }} transfer.json
+
+      - name: Upload data to artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: transfer-page-${{ matrix.page }}
+          path: transfer_scraper/transfer.json
+