quatalog-scraper/.github/workflows/transfer.yml

name: Scrape transfer and update file
run-name: Scrape transfer and update file
on:
  # schedule:
  # - cron: '*/15 * * * *'
  repository_dispatch:
    types: transfer-scraper
  workflow_dispatch:
    inputs:
      timeout:
        description: "Timeout time"
        required: true
        type: number
        default: 120
concurrency:
  group: transfer-scraper

jobs:
  scrape-data:
    name: Scrape transfer guide
    runs-on: ubuntu-latest
    steps:
      - name: Checkout scraping repo
        uses: actions/checkout@v4
        with:
          path: quatalog-scraping

      - name: Checkout data repo
        uses: actions/checkout@v4
        with:
          repository: quatalog/data
          path: data

      - name: Set up python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Install dependencies
        working-directory: quatalog-scraping/transfer_scraper
        run: |
          python -m pip install --upgrade pip
          pip install -r 'requirements.txt'

      - name: Log IP
        run: |
          echo "Public IP: $(curl -s 'https://ipinfo.io/ip')"

      - name: Scrape transfer guide
        run: |
          mkdir new-data
          rsync -avzh data/transfer.json new-data
          rsync -avzh data/transfer_state.json new-data
          python3 quatalog-scraping/transfer_scraper/main.py new-data/transfer.json new-data/transfer_state.json ${{ github.event.inputs.timeout }}

      - name: Upload data to artifact
        uses: actions/upload-artifact@v4
        with:
          name: transfer-data
          path: new-data/

  push-new-data:
    name: Push new data to data repo
    runs-on: ubuntu-latest
    needs: [scrape-data]
    steps:
      - name: Clone Quatalog data
        uses: actions/checkout@v4
        with:
          repository: quatalog/data
          path: quatalog-data
          token: ${{ secrets.PUSH_TOKEN }}

      - name: Download data from artifact
        uses: actions/download-artifact@v4
        with:
          name: transfer-data
          path: data

      - name: Copy data to repo directory
        run: |
          rsync -avzh data/ quatalog-data/

      - name: Push new data
        working-directory: quatalog-data
        run: |
          git config user.name "Quatalog Updater"
          git config user.email "github_actions@quatalog.com"
          git add transfer.json transfer_state.json
          git commit -m "$(date)" || exit 0
          git push

  re-run-scraper:
    name: Tell Github to run this workflow again
    runs-on: ubuntu-latest
    needs: [push-new-data]
    steps:
      - name: Tell Github to run this workflow again
        run: |
          curl -L \
            -H "Accept: application/vnd.github+json" \
            -H "Authorization: token ${{ secrets.PUSH_TOKEN }}" \
            --request POST \
            --data '{"event_type": "transfer-scraper"}' \
            "https://api.github.com/repos/quatalog/quatalog/dispatches"