Merge branch 'main' of https://github.com/quatalog/quatalog-scraping

.github/workflows/scraper.yml

@@ -1,4 +1,4 @@
-name: Update data
+name: Update data and generate static site
 on:
   workflow_dispatch:
   schedule:
@@ -20,63 +20,101 @@ jobs:
           repository: quacs/quacs-data
           path: quacs-data
 
+      - name: Run scraper
+        run: |
+          # Usage: CourseOfferingsScraper <data_directory> <terms_offered_file> <prerequisites_file> <list_of_terms_file>
+          mkdir new-data
+          quatalog-scraping/bin/CourseOfferingsScraper \
+            quacs-data/semester_data \
+            new-data/terms_offered.json \
+            new-data/prerequisites.json \
+            new-data/terms_list.json
+            
+      - name: Copy catalog.json from QuACS data
+        run: |
+          CURRENT_TERM="$(grep new-data/terms_list.json -e 'current_term' | cut -f4 -d '"')"
+          rsync -avz "quacs-data/semester_data/$CURRENT_TERM/catalog.json" new-data/catalog.json
+          
+      - name: Upload data to artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: new-data
+          path: new-data/
+            
+  push-new-data:
+    name: Push new data to data repo
+    runs-on: ubuntu-latest
+    needs: [scrape-data]
+    steps:
       - name: Clone Quatalog data
         uses: actions/checkout@v3
         with:
           repository: quatalog/data
           path: quatalog-data
           token: ${{ secrets.PUSH_TOKEN }}
-
+    
-      - name: Run scraper
-        run: |
-          # Usage: CourseOfferingsScraper <data_directory> <terms_offered_file> <prerequisites_file> <list_of_terms_file>
-          quatalog-scraping/bin/CourseOfferingsScraper \
-            quacs-data/semester_data \
-            quatalog-data/terms_offered.json \
-            quatalog-data/prerequisites.json \
-            quatalog-data/terms_list.json
-            
-      - name: Copy catalog.json from QuACS data
-        run: |
-          CURRENT_TERM="$(grep quatalog-data/terms_list.json -e 'current_term' | cut -f4 -d '"')"
-          rsync -avz "quacs-data/semester_data/$CURRENT_TERM/catalog.json" quatalog-data/catalog.json
-          
-      - name: List files
-        run: |
-          ls -lsaR .
-          
-      - name: Upload data to artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: quatalog-data
-          path: quatalog-data/
-
-
-#  generate-site:
-#    runs-on: ubuntu-latest
-#    needs: [scrape-data]
-#    steps:
-#      - name: Checkout scraping repo
-#        uses: actions/checkout@v3
-#        with:
-#          path: quatalog-scraping
-          
-  push-new-data:
-    name: Push new data to data repo
-    runs-on: ubuntu-latest
-    needs: [scrape-data]
-    steps:
       - name: Download data from artifact
         uses: actions/download-artifact@v3
         with:
-          name: quatalog-data
+          name: new-data
-          path: quatalog-data
+          path: new-data
+      
+      - name: Copy data to repo directory
+        run: |
+          rsync -avz new-data/ quatalog-data/
           
       - name: Push new data
-        working-directory: ./quatalog-data
+        working-directory: quatalog-data
         run: |
           git config --global user.name "Quatalog Updater"
           git config --global user.email "github_actions@quatalog.com"
           git add terms_offered.json prerequisites.json terms_list.json catalog.json
           git commit -m "$(date)" || exit 0
           git push
+
+  generate-site:
+    name: Generate the static site
+    runs-on: ubuntu-latest
+    needs: [scrape-data]
+    steps:
+      - name: Checkout scraping repo
+        uses: actions/checkout@v3
+        with:
+          path: quatalog-scraping
+      
+      - name: Clone Quatalog static site
+        uses: actions/checkout@v3
+        with:
+          repository: quatalog/site
+          ref: static-generated
+          path: quatalog-site
+          token: ${{ secrets.PUSH_TOKEN }}
+      
+      - name: Download data from artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: new-data
+          path: new-data
+          
+      - name: Run the HTML generator
+        run: |
+          # Usage: GenerateHtml <terms_offered_file> <prerequisites_file> <list_of_terms_file> <catalog_file> <out_directory>
+          quatalog-scraping/bin/GenerateHtml \
+            new-data/terms_offered.json \
+            new-data/prerequisites.json \
+            new-data/terms_list.json \
+            new-data/catalog.json \
+            courses/
+            
+      - name: Merge data
+        run: |
+          rsync -avz courses/ quatalog-site/courses/
+        
+      - name: Push generated HTML
+        working-directory: quatalog-site
+        run: |
+          git config --global user.name "Quatalog Updater"
+          git config --global user.email "github_actions@quatalog.com"
+          git add courses
+          git commit -m "$(date)" || exit 0
+          git push