quatalog-scraper/.github/workflows/transfer.yml

86 lines
2.2 KiB
YAML
Raw Normal View History

2024-02-29 22:16:39 +00:00
name: Scrape transfer and update file
on:
2024-03-01 02:10:19 +00:00
# schedule:
# - cron: '15 * * * *'
2024-02-29 22:16:39 +00:00
workflow_dispatch:
2024-03-01 02:10:19 +00:00
inputs:
timeout:
description: "Timeout time"
required: true
type: number
default: 120
2024-02-29 22:16:39 +00:00
concurrency:
group: transfer-scraper
jobs:
2024-03-01 01:37:31 +00:00
scrape-data:
2024-02-29 22:16:39 +00:00
name: Scrape transfer guide
runs-on: ubuntu-latest
steps:
- name: Checkout scraping repo
uses: actions/checkout@v4
with:
path: quatalog-scraping
2024-03-01 01:37:31 +00:00
- name: Checkout data repo
uses: actions/checkout@v4
with:
path: data
2024-02-29 22:16:39 +00:00
- name: Set up python
uses: actions/setup-python@v4
with:
python-version: '3.11'
cache: 'pip'
- name: Install dependencies
working-directory: quatalog-scraping/transfer_scraper
run: |
python -m pip install --upgrade pip
pip install -r 'requirements.txt'
2024-02-29 22:16:39 +00:00
- name: Log IP
2024-03-01 01:37:31 +00:00
run: |
echo "Public IP: $(curl -s 'https://ipinfo.io/ip')"
2024-02-29 22:16:39 +00:00
- name: Scrape transfer guide
run: |
2024-03-01 02:10:19 +00:00
python3 quatalog-scraping/transfer_scraper/main.py data/transfer.json data/transfer_state.json ${{ github.event.inputs.timeout }}
2024-02-29 22:16:39 +00:00
- name: Upload data to artifact
uses: actions/upload-artifact@v4
with:
name: transfer-data
path: data/
push-new-data:
name: Push new data to data repo
runs-on: ubuntu-latest
needs: [scrape-data]
steps:
- name: Clone Quatalog data
2024-03-01 02:10:19 +00:00
uses: actions/checkout@v4
2024-02-29 22:16:39 +00:00
with:
repository: quatalog/data
path: quatalog-data
token: ${{ secrets.PUSH_TOKEN }}
- name: Download data from artifact
2024-03-01 02:10:19 +00:00
uses: actions/download-artifact@v4
2024-02-29 22:16:39 +00:00
with:
name: transfer-data
path: data/
- name: Copy data to repo directory
run: |
rsync -avz data/ quatalog-data/
- name: Push new data
working-directory: quatalog-data
run: |
git config user.name "Quatalog Updater"
git config user.email "github_actions@quatalog.com"
git add transfer.json transfer_state.json
git commit -m "$(date)" || exit 0
git push