diff --git a/.github/workflows/scraper.yml b/.github/workflows/scraper.yml index dd2097d..a8faedb 100644 --- a/.github/workflows/scraper.yml +++ b/.github/workflows/scraper.yml @@ -66,8 +66,8 @@ jobs: - name: Push new data working-directory: quatalog-data run: | - git config --global user.name "Quatalog Updater" - git config --global user.email "github_actions@quatalog.com" + git config user.name "Quatalog Updater" + git config user.email "github_actions@quatalog.com" git add terms_offered.json prerequisites.json terms_list.json catalog.json git commit -m "$(date)" || exit 0 git push @@ -123,6 +123,42 @@ jobs: git commit -m "$(date)" || exit 0 git push + push-csv: + name: Push CSV file to data repo + runs-on: ubuntu-latest + needs: [generate-site] + steps: + - name: Checkout site repo/static-generated branch + uses: actions/checkout@v3 + with: + repository: quatalog/site + ref: static-generated + path: static-generated + + - name: Checkout data repo + uses: actions/checkout@v3 + with: + repository: quatalog/data + path: quatalog-data + token: ${{ secrets.PUSH_TOKEN }} + + - name: Create CSV file + run: | + grep -oe '
  • .*
  • ' static-generated/courses/*.html | + sort -u | + grep -ve "-[0-9]9[4-7][0-9]" -e "-[0-9]9[89]0" -e "-[0-9]000[1-9]" -e "USAR" -e "ADMN" -e "USNA" -e "USAF" | + sed -e 's/^/"/' -e 's/.html:
  • /","/' -e 's#
  • #"#' -e 's/"\([^"]*\)","\([^"]*\)"/"\2","\1"/' | + sort > quatalog-data/courses.csv + + - name: Push CSV file + working-directory: quatalog-data + run: | + git config user.name "Quatalog Updater" + git config user.email "github_actions@quatalog.com" + git add courses.csv + git commit -m "$(date)" || exit 0 + git push + prepare-site: name: Prepare static site runs-on: ubuntu-latest