diff --git a/.github/workflows/scraper.yml b/.github/workflows/scraper.yml index aacb502..361ab6f 100644 --- a/.github/workflows/scraper.yml +++ b/.github/workflows/scraper.yml @@ -1,4 +1,4 @@ -name: Update data +name: Update data and generate static site on: workflow_dispatch: schedule: @@ -20,63 +20,101 @@ jobs: repository: quacs/quacs-data path: quacs-data + - name: Run scraper + run: | + # Usage: CourseOfferingsScraper + mkdir new-data + quatalog-scraping/bin/CourseOfferingsScraper \ + quacs-data/semester_data \ + new-data/terms_offered.json \ + new-data/prerequisites.json \ + new-data/terms_list.json + + - name: Copy catalog.json from QuACS data + run: | + CURRENT_TERM="$(grep new-data/terms_list.json -e 'current_term' | cut -f4 -d '"')" + rsync -avz "quacs-data/semester_data/$CURRENT_TERM/catalog.json" new-data/catalog.json + + - name: Upload data to artifact + uses: actions/upload-artifact@v3 + with: + name: new-data + path: new-data/ + + push-new-data: + name: Push new data to data repo + runs-on: ubuntu-latest + needs: [scrape-data] + steps: - name: Clone Quatalog data uses: actions/checkout@v3 with: repository: quatalog/data path: quatalog-data token: ${{ secrets.PUSH_TOKEN }} - - - name: Run scraper - run: | - # Usage: CourseOfferingsScraper - quatalog-scraping/bin/CourseOfferingsScraper \ - quacs-data/semester_data \ - quatalog-data/terms_offered.json \ - quatalog-data/prerequisites.json \ - quatalog-data/terms_list.json - - - name: Copy catalog.json from QuACS data - run: | - CURRENT_TERM="$(grep quatalog-data/terms_list.json -e 'current_term' | cut -f4 -d '"')" - rsync -avz "quacs-data/semester_data/$CURRENT_TERM/catalog.json" quatalog-data/catalog.json - - - name: List files - run: | - ls -lsaR . - - - name: Upload data to artifact - uses: actions/upload-artifact@v3 - with: - name: quatalog-data - path: quatalog-data/ - - -# generate-site: -# runs-on: ubuntu-latest -# needs: [scrape-data] -# steps: -# - name: Checkout scraping repo -# uses: actions/checkout@v3 -# with: -# path: quatalog-scraping - - push-new-data: - name: Push new data to data repo - runs-on: ubuntu-latest - needs: [scrape-data] - steps: + - name: Download data from artifact uses: actions/download-artifact@v3 with: - name: quatalog-data - path: quatalog-data + name: new-data + path: new-data + + - name: Copy data to repo directory + run: | + rsync -avz new-data/ quatalog-data/ - name: Push new data - working-directory: ./quatalog-data + working-directory: quatalog-data run: | git config --global user.name "Quatalog Updater" git config --global user.email "github_actions@quatalog.com" git add terms_offered.json prerequisites.json terms_list.json catalog.json git commit -m "$(date)" || exit 0 git push + + generate-site: + name: Generate the static site + runs-on: ubuntu-latest + needs: [scrape-data] + steps: + - name: Checkout scraping repo + uses: actions/checkout@v3 + with: + path: quatalog-scraping + + - name: Clone Quatalog static site + uses: actions/checkout@v3 + with: + repository: quatalog/site + ref: static-generated + path: quatalog-site + token: ${{ secrets.PUSH_TOKEN }} + + - name: Download data from artifact + uses: actions/download-artifact@v3 + with: + name: new-data + path: new-data + + - name: Run the HTML generator + run: | + # Usage: GenerateHtml + quatalog-scraping/bin/GenerateHtml \ + new-data/terms_offered.json \ + new-data/prerequisites.json \ + new-data/terms_list.json \ + new-data/catalog.json \ + courses/ + + - name: Merge data + run: | + rsync -avz courses/ quatalog-site/courses/ + + - name: Push generated HTML + working-directory: quatalog-site + run: | + git config --global user.name "Quatalog Updater" + git config --global user.email "github_actions@quatalog.com" + git add courses + git commit -m "$(date)" || exit 0 + git push diff --git a/bin/CourseOfferingsScraper b/bin/CourseOfferingsScraper index 62cccd4..ea234b1 100755 Binary files a/bin/CourseOfferingsScraper and b/bin/CourseOfferingsScraper differ diff --git a/bin/GenerateHtml b/bin/GenerateHtml index bbbabb9..a570669 100755 Binary files a/bin/GenerateHtml and b/bin/GenerateHtml differ diff --git a/bin/json.o b/bin/json.o index 8c3e582..832cf80 100644 Binary files a/bin/json.o and b/bin/json.o differ