phantop
/
quatalog-scraper
mirror of https://github.com/quatalog/quatalog.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: phantop:0109006d129bcc08d8d6c20743359b54307c000d
Branches Tags
phantop:main
...
compare: phantop:be754b5dcb4b81bcb326359790eacfe44c4a0afc
Branches Tags
phantop:main

3 Commits
0109006d12 ... be754b5dcb

Author SHA1 Message Date
powe97 be754b5dcb
Update transfer.yml 2024-04-08 14:33:44 -04:00
powe97 c7af001773
Update transfer.yml 2024-04-08 14:32:48 -04:00
powe97 3b16ecc9bb
Update transfer.yml 2024-04-08 14:23:51 -04:00
1 changed files with 6 additions and 1 deletions
Show Stats Expand all files Collapse all files

7
.github/workflows/transfer.yml vendored
View File

@ -13,7 +13,12 @@ jobs:
- name: Create matrix parameters
id: matrix-params
run: |
NUM_PAGES="$(curl -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:122.0) Gecko/20100101 Firefox/122.0' 'https://tes.collegesource.com/publicview/TES_publicview01.aspx?rid=f080a477-bff8-46df-a5b2-25e9affdd4ed&aid=27b576bb-cd07-4e57-84d0-37475fde70ce' | grep -e 'lblInstWithEQPaginationInfo' | grep -Poie '(?<=of )[0-9]*')"
UA='User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:122.0) Gecko/20100101 Firefox/122.0'
URL='https://tes.collegesource.com/publicview/TES_publicview01.aspx?rid=f080a477-bff8-46df-a5b2-25e9affdd4ed&aid=27b576bb-cd07-4e57-84d0-37475fde70ce'
HTML="$(curl -s -H "$UA" "$URL")"
echo "$HTML" | grep -A2 "<title>"
NUM_PAGES="$(echo "$HTML" | grep -e 'lblInstWithEQPaginationInfo' | grep -Poie '(?<=of )[0-9]*')"
echo "Found $NUM_PAGES pages"
MATRIX_PARAMS="$(seq -s "," 1 "$NUM_PAGES")"
MATRIX_PARAMS="\"page\": $(sed -e 's/,/}, {"page": /g' <<< "$MATRIX_PARAMS")"
echo "matrix-params={\"include\": [{"$MATRIX_PARAMS"}]}" | tee $GITHUB_OUTPUT