phantop/quatalog-scraper
1
0
Fork 0
mirror of https://github.com/quatalog/quatalog.git synced 2025-03-27 04:19:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Changes to make generating HTML easier

Browse source
This commit is contained in:
3eef8a28f26fb2bcc514e6f1938929a1f931762 2023-02-07 18:10:56 -05:00
parent 2bc605006e
commit e35aa41b4a
3 changed files with 178 additions and 24 deletions

48
src/CourseOfferingsScraper.cpp
View file

@ -5,17 +5,17 @@
#include<filesystem>
#include<unordered_set>
#include<json/json.h>
#include"common.h"
namespace fs = std::filesystem;
struct term_data_t {
Json::Value courses;
Json::Value prerequisites;
};
struct quatalog_data_t {
Json::Value terms_offered;
Json::Value prerequisites;
Json::Value list_of_terms;
};
struct term_data_t {
Json::Value courses;
Json::Value prerequisites;
};
using course_handler_t = void(const Json::Value&,const std::string&,quatalog_data_t&,const Json::Value&);
void handle_term_dirs(const std::set<fs::directory_entry>&,quatalog_data_t&);
@ -23,7 +23,7 @@ void handle_term(const fs::directory_entry& term_entry,quatalog_data_t&);
void handle_prefix(const Json::Value&,const std::string&,quatalog_data_t&,const term_data_t&,course_handler_t*);
void handle_course(const Json::Value&,const std::string&,quatalog_data_t&,const Json::Value&);
void handle_course_summer(const Json::Value&,const std::string&,quatalog_data_t&,const Json::Value&);
void handle_everything(const Json::Value&,const Json::Value&,Json::Value& course_term,Json::Value&,const Json::Value&);
void handle_everything(const Json::Value&,const Json::Value&,const std::string&,Json::Value& course_term,Json::Value&,const Json::Value&);
void handle_sections(const Json::Value&,Json::Value&);
void handle_instructors(const Json::Value&,std::unordered_set<std::string>&);
void handle_multiple_instructors(const std::string&,std::unordered_set<std::string>&);
@ -148,9 +148,9 @@ void handle_course(const Json::Value& course,
quatalog_data_t& data,
const Json::Value& term_prereqs) {
const auto& course_code = course["id"].asString();
auto& course_term = data.terms_offered[course_code][term];
auto& course_terms = data.terms_offered[course_code];
const Json::Value& sections = course["sections"];
handle_everything(sections,course,course_term,data.prerequisites,term_prereqs);
handle_everything(sections,course,term,course_terms,data.prerequisites,term_prereqs);
}
void handle_course_summer(const Json::Value& course,
@ -196,7 +196,8 @@ void handle_course_summer(const Json::Value& course,
if(subterm0) {
handle_everything(sections[0],
course,
course_terms[term],
term,
course_terms,
data.prerequisites,
term_prereqs);
return;
@ -204,14 +205,16 @@ void handle_course_summer(const Json::Value& course,
if(subterm1) {
handle_everything(sections[1],
course,
course_terms[term+"02"],
term+"02",
course_terms,
data.prerequisites,
term_prereqs);
}
if(subterm2) {
handle_everything(sections[2],
course,
course_terms[term+"03"],
term+"03",
course_terms,
data.prerequisites,
term_prereqs);
}
@ -219,13 +222,17 @@ void handle_course_summer(const Json::Value& course,
void handle_everything(const Json::Value& sections,
const Json::Value& course,
Json::Value& course_term,
const std::string& term,
Json::Value& course_terms,
Json::Value& out_prereqs,
const Json::Value& term_prereqs) {
Json::Value& course_term = course_terms[term];
const auto& course_id = course["id"].asString();
course_term["title"] = course["title"];
handle_sections(sections,course_term);
handle_attributes(sections[0],course["id"].asString(),course_term,out_prereqs);
handle_prereqs(sections[0],course["id"].asString(),out_prereqs,term_prereqs);
course_terms["latest_term"] = term;
handle_attributes(sections[0],course_id,course_term,out_prereqs);
handle_prereqs(sections[0],course_id,out_prereqs,term_prereqs);
}
void handle_sections(const Json::Value& sections,
@ -354,14 +361,7 @@ void handle_prereqs(const Json::Value& section,
const auto& prerequisites = in_obj["prerequisites"];
const auto& cross_listings = in_obj["cross_list_courses"];
// Scraper does not work as intended if we use
// a variable instead of repeating out_data[course_id]
// This would result in null entries for courses that
// have major restrictions or something else like that
if(!corequisites.empty())
out_data[course_id]["corequisites"] = corequisites;
if(!prerequisites.empty())
out_data[course_id]["prerequisites"] = prerequisites;
if(!cross_listings.empty())
out_data[course_id]["cross_listings"] = cross_listings;
out_data[course_id]["corequisites"] = corequisites;
out_data[course_id]["prerequisites"] = prerequisites;
out_data[course_id]["cross_listings"] = cross_listings;
}

153
src/GenerateHtml.cpp Normal file
View file

@ -0,0 +1,153 @@
#include<fstream>
#include<iostream>
#include<filesystem>
#include<json/json.h>
#include"common.h"
struct quatalog_data_t {
Json::Value terms_offered;
Json::Value prerequisites;
Json::Value list_of_terms;
Json::Value catalog;
};
bool create_dir_if_not_exist(const fs::path&);
void generate_course_page(const std::string&,const quatalog_data_t&,std::ostream&);
void generate_list(const Json::Value&,const std::string&,const std::string&,const quatalog_data_t&,std::ostream& os);
std::string get_course_title(const std::string&,const quatalog_data_t&);
int main(const int argc,
const char** argv) {
if(argc != 6) {
std::cerr << "Bad number of arguments (" << argc << ")" << std::endl;
std::cerr << "Usage: " << argv[0]
<< " <terms_offered_file>"
<< " <prerequisites_file>"
<< " <list_of_terms_file>"
<< " <catalog_file>"
<< " <out_directory>"
<< std::endl;
return EXIT_FAILURE;
}
const auto& terms_offered_filename = std::string(argv[1]);
const auto& prerequisites_filename = std::string(argv[2]);
const auto& list_of_terms_filename = std::string(argv[3]);
const auto& catalog_filename = std::string(argv[4]);
const auto& out_dir_path = fs::path(argv[5]);
if(!create_dir_if_not_exist(out_dir_path)) {
std::cerr << "What" << std::endl;
return EXIT_FAILURE;
}
std::fstream terms_offered_file{terms_offered_filename,std::ios::in};
std::fstream prerequisites_file{prerequisites_filename,std::ios::in};
std::fstream list_of_terms_file{list_of_terms_filename,std::ios::in};
std::fstream catalog_file{catalog_filename,std::ios::in};
quatalog_data_t quatalog_data;
terms_offered_file >> quatalog_data.terms_offered;
prerequisites_file >> quatalog_data.prerequisites;
list_of_terms_file >> quatalog_data.list_of_terms;
catalog_file >> quatalog_data.catalog;
generate_course_page("LGHT-4830",quatalog_data,std::cout);
generate_course_page("MATH-4150",quatalog_data,std::cout);
}
bool create_dir_if_not_exist(const fs::path& path) {
if(fs::exists(path)) {
if(!fs::is_directory(path)) {
std::cerr << "out_directory argument "
<< path
<< "is not a directory"
<< std::endl;
return false;
}
return true;
}
return fs::create_directory(path);
}
void generate_course_page(const std::string& course_id,
const quatalog_data_t& quatalog_data,
std::ostream& os) {
std::string course_name, description;
const auto& catalog_entry = quatalog_data.catalog[course_id];
const auto& prereqs_entry = quatalog_data.prerequisites[course_id];
const auto& terms_offered = quatalog_data.terms_offered[course_id];
const auto& latest_term = terms_offered["latest_term"].asString();
const auto& credits = terms_offered[latest_term]["credits"];
const int credMin = credits["min"].asInt();
const int credMax = credits["max"].asInt();
const auto& credit_string = (credMin == credMax) ? std::to_string(credMin) : (std::to_string(credMin) + "-" + std::to_string(credMax));
if(catalog_entry) {
course_name = catalog_entry["name"].asString();
description = catalog_entry["description"].asString();
} else {
course_name = terms_offered[latest_term]["name"].asString();
description = "This course is not in the most recent catalog."
"It may have been discontinued, or had its course "
"code changed.";
}
const auto& mid_digits = course_id.substr(6,2);
if(mid_digits == "96" || mid_digits == "97") {
course_name = "Topics in " + course_id.substr(0,4);
description = "Course codes between X960 and X979 are for topics courses. "
"They are often recycled and used for new/experimental courses.";
}
os << R"R(<html>
<head>
<title>)R" << course_id << " - " << course_name << R"R(</title>"
</head>
<body>
<div id="cd-flex">
<div id="course-info-container">
<h1 id="name">)R" << course_name << R"R(</h1>
<h2 id="code">)R" << course_id << R"R(</h2>
<p>)R" << description << R"R(</p>
<div id="cattrs-container">
<span id="credits-pill" class="attr-pill">
<span>)R" << credit_string << " " << (credMax == 1 ? "credit" : "credits") << R"R(</span>
</span>
</div>)R" << '\n';
generate_list(prereqs_entry["cross_listings"],"Cross-listed with:","crosslist",quatalog_data,os);
generate_list(prereqs_entry["corequisites"],"Corequisites:","coreq",quatalog_data,os);
os << R"R( </div>
</div>
</body>
</html>)R" << std::endl;
}
std::string get_course_title(const std::string& course_id,const quatalog_data_t& quatalog_data) {
const auto& catalog_entry = quatalog_data.catalog[course_id];
const auto& terms_offered = quatalog_data.terms_offered[course_id];
const auto& latest_term = terms_offered["latest_term"].asString();
if(catalog_entry) {
return catalog_entry["name"].asString();
} else {
return terms_offered[latest_term]["name"].asString();
}
}
void generate_list(const Json::Value& list,const std::string& list_name,const std::string& css_prefix,const quatalog_data_t& qlog,std::ostream& os) {
if(!list.empty()) {
os << R"R( <div id=")R" << css_prefix << R"R(-container" class="hidden">
<div id=")R" << css_prefix << R"R(-title" class="rel-info-title">
)R" << list_name << R"R(
</div>
<div id=")R" << css_prefix << R"R(-classes" class="rel-info-courses">)R";
for(const auto& cl : list) {
const auto& clstr = cl.asString();
os << R"R(
<a class="course-pill" href=")R" << clstr
<< R"R(.html">)R"
<< clstr << " " << get_course_title(clstr,qlog)
<< "</a>\n";
}
os << " </div>\n";
os << " </div>\n";
}
}

1
src/common.h Normal file
View file

@ -0,0 +1 @@
namespace fs = std::filesystem;