phantop/quatalog-scraper
1
0
Fork 0
mirror of https://github.com/quatalog/quatalog.git synced 2024-11-16 03:52:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Lint

Browse source
This commit is contained in:
powe97 2024-03-16 01:17:49 -04:00
parent c3be28e520
commit ba3e5c77d1
No known key found for this signature in database
GPG key ID: 7D1663B10978D1BA
2 changed files with 9 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
transfer_scraper/convert_json.py
View file

@ -2,9 +2,13 @@ import json
import sys
import collections
def main():
if len(sys.argv) != 3:
print(f"USAGE: python {sys.argv[0]} <json from scraper> <by-course output file>", file=sys.stderr)
print(
f"USAGE: python {sys.argv[0]} <json from scraper> <by-course output file>",
file=sys.stderr,
)
return 1
with open(sys.argv[1], "r") as scraper_json:
by_institution = json.load(scraper_json)
@ -20,5 +24,6 @@ def main():
with open(sys.argv[2], "w") as out_json:
json.dump(by_rpi_course, out_json, sort_keys=True, indent=2)
if __name__ == "__main__":
exit(main())

4
transfer_scraper/scrape_page.py
View file

@ -306,7 +306,9 @@ def main():
print(f"Using randomized user agent {user_agent}", file=sys.stderr)
with open(OUT_FILENAME, "w") as transferjson:
json.dump(scrape_page(PAGE_NUM_TO_SCRAPE), transferjson, sort_keys=True, indent=2)
json.dump(
scrape_page(PAGE_NUM_TO_SCRAPE), transferjson, sort_keys=True, indent=2
)
driver.quit()