diff --git a/.github/workflows/transfer.yml b/.github/workflows/transfer.yml index 4468519..e811375 100644 --- a/.github/workflows/transfer.yml +++ b/.github/workflows/transfer.yml @@ -17,6 +17,7 @@ jobs: echo "matrix-params={\"include\": [{"$MATRIX_PARAMS"}]}" | tee $GITHUB_OUTPUT outputs: matrix-params: ${{ steps.matrix-params.outputs.matrix-params }} + scrape-page: name: Scrape page runs-on: ubuntu-latest @@ -51,3 +52,33 @@ jobs: name: transfer-page-${{ matrix.page }} path: transfer_scraper/transfer_${{ matrix.page }}.json + commit-data: + name: Combine and commit data + runs-on: ubuntu-latest + needs: scrape-page + steps: + - name: Checkout data repo + uses: actions/checkout@v4 + with: + repository: quatalog/data + path: data + + - name: Download partial JSONs + uses: actions/download-artifact@v4 + with: + pattern: transfer-page-* + merge-multiple: true + path: new-data + + - name: Combine JSONs + run: | + cat new-data/* | jq -s | tee data/transfer.json + + - name: Commit data + working-directory: data + run: | + git config user.name "Quatalog Updater" + git config user.email "github_actions@quatalog.com" + git add transfer.json + git commit -m "$(date)" || exit 0 + git push