phantop/quatalog-scraper
1
0
Fork 0
mirror of https://github.com/quatalog/quatalog.git synced 2024-11-08 08:04:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix summer term stuff. It's a mess

Browse source
This commit is contained in:
3eef8a28f26fb2bcc514e6f1938929a1f931762 2023-02-04 17:29:50 -05:00
parent d6dc6b080c
commit 7b10a24577
1 changed files with 112 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

151
CourseOfferingsScraper.cpp
View file

@ -6,9 +6,12 @@
#include<dist/jsoncpp.cpp>
namespace fs = std::filesystem;
using course_handler_t = void(const Json::Value&,const std::string&,Json::Value&);
void handle_term(const fs::directory_entry&,Json::Value&);
void handle_prefix(const Json::Value&,const std::string&,Json::Value&);
void handle_term_dirs(const std::set<fs::directory_entry>&,Json::Value&);
void handle_prefix(const Json::Value&,const std::string&,Json::Value&,course_handler_t*);
void handle_course(const Json::Value&,const std::string&,Json::Value&);
void handle_course_summer(const Json::Value&,const std::string&,Json::Value&);
void handle_attribute(const std::string&,Json::Value&);
void handle_attributes(const Json::Value&,Json::Value&);
void handle_sections(const Json::Value&,Json::Value&);
@ -26,15 +29,25 @@ int main(const int argc,const char** argv) {
return EXIT_FAILURE;
}
const auto data_dir = fs::directory_iterator(data_dir_path);
// Sort term dirs chronologically using a std::set
std::set<fs::directory_entry> term_dirs;
for(const fs::directory_entry term : data_dir) {
const auto data_dir = fs::directory_iterator(data_dir_path);
for(const auto& term : data_dir) {
term_dirs.insert(term);
}
// Begin JSON manipulation
Json::Value terms_offered;
terms_offered["all_terms"] = Json::arrayValue;
handle_term_dirs(term_dirs,terms_offered);
std::cout << terms_offered << std::endl;
return EXIT_SUCCESS;
}
void handle_term_dirs(const std::set<fs::directory_entry>& term_dirs,
Json::Value& terms_offered) {
for(auto term : term_dirs) {
if(!fs::is_directory(term)) {
continue;
@ -43,15 +56,10 @@ int main(const int argc,const char** argv) {
handle_term(term,terms_offered);
}
//std::cout << terms_offered["all_terms"] << std::endl;
//std::cout << terms_offered["MATH-4020"] << std::endl;
std::cout << terms_offered << std::endl;
return EXIT_SUCCESS;
}
void handle_term(const fs::directory_entry& term_entry,
Json::Value& terms_offered) {
Json::Value& terms_offered) {
const fs::path dir = term_entry.path();
const auto dirname = dir.string();
const auto term = dir.stem().string();
@ -60,21 +68,27 @@ void handle_term(const fs::directory_entry& term_entry,
std::fstream courses_file{courses_filename,std::ios::in};
std::fstream prereqs_file{prereqs_filename,std::ios::in};
#ifdef DEBUG
std::cerr << '\t' << courses_filename << std::endl;
std::cerr << '\t' << prereqs_filename << std::endl;
#endif
std::cerr << "Processing term " << term << "..." << std::endl;
terms_offered["all_terms"].append(term);
Json::Value courses;
courses_file >> courses;
for(auto prefix : courses) {
handle_prefix(prefix,term,terms_offered);
course_handler_t* course_handler;
if(term.substr(4,2) == "05") {
course_handler = handle_course_summer;
} else {
course_handler = handle_course;
}
for(auto& prefix : courses) {
handle_prefix(prefix,term,terms_offered,course_handler);
}
}
void handle_prefix(const Json::Value& prefix,const std::string& term,Json::Value& terms_offered) {
void handle_prefix(const Json::Value& prefix,
const std::string& term,
Json::Value& terms_offered,
course_handler_t course_handler) {
#ifdef DEBUG
std::cerr << "\tProcessing prefix "
<< prefix["code"]
@ -83,27 +97,91 @@ void handle_prefix(const Json::Value& prefix,const std::string& term,Json::Value
<< "..."
<< std::endl;
#endif
for(auto course : prefix["courses"]) {
handle_course(course,term,terms_offered);
for(auto& course : prefix["courses"]) {
course_handler(course,term,terms_offered);
}
}
void handle_course(const Json::Value& course,const std::string& term,Json::Value& terms_offered) {
void handle_course(const Json::Value& course,
const std::string& term,
Json::Value& terms_offered) {
const auto course_code = course["id"].asString();
Json::Value sections = course["sections"];
auto& course_term = terms_offered[course_code][term];
course_term["title"] = course["title"];
const Json::Value& sections = course["sections"];
handle_sections(sections,course_term);
handle_attributes(course["sections"][0],course_term);
}
void handle_sections(const Json::Value& sections,Json::Value& course_term) {
void handle_course_summer(const Json::Value& course,
const std::string& term,
Json::Value& terms_offered) {
const auto course_code = course["id"].asString();
// sections[0]: Full term sections
// sections[1]: First-half term sections
// sections[2]: Second-half term sections
// Of course, a course will never be offered
// in both the full term _and_ one of the
// half-terms, but there are a few that
// are offered in both halves (e.g. STSO-4100)
std::array<Json::Value,3> sections;
// We will loop twice over the sections,
// once here and once in handle_sections,
// but frankly I tried to make it all in 1
// loop and the code was a total unreadable
// mess. So I don't really care
int subterm;
bool subterm0 = false, subterm1 = false, subterm2 = false;
for(const auto& section : course["sections"]) {
const auto& timeslot = section["timeslots"][0];
const std::string& dateEnd = timeslot["dateEnd"].asString();
const std::string& dateStart = timeslot["dateStart"].asString();
subterm = 0;
if(dateStart.substr(0,2) != "05") {
subterm = 1;
subterm1 = true;
} else if(dateEnd.substr(0,2) != "08") {
subterm = 2;
subterm2 = true;
} else {
subterm0 = true;
}
sections[subterm].append(section);
}
auto& course_terms = terms_offered[course_code];
// Forgive the repeated code; it's not that much of a mess
if(subterm0) {
Json::Value& course_term = course_terms[term];
course_term["title"] = course["title"];
handle_sections(sections[0],course_term);
handle_attributes(sections[0][0],course_term);
} else {
if(subterm1) {
Json::Value& course_term = course_terms[term+"02"];
course_term["title"] = course["title"];
handle_sections(sections[1],course_term);
handle_attributes(sections[1][0],course_term);
} if(subterm2) {
Json::Value& course_term = course_terms[term+"03"];
course_term["title"] = course["title"];
handle_sections(sections[2],course_term);
handle_attributes(sections[2][0],course_term);
}
}
}
void handle_sections(const Json::Value& sections,
Json::Value& course_term) {
int credMin = Json::Value::maxInt, credMax = 0;
int seatsTaken = 0, capacity = 0, remaining = 0;
for(auto section : sections) {
for(const auto& section : sections) {
// Get min/max credits *of all sections*
// (RCOS looking at you)
credMin = std::min(credMin,section["credMin"].asInt());
@ -125,10 +203,8 @@ void handle_sections(const Json::Value& sections,Json::Value& course_term) {
course_term["seats"]["remaining"] = remaining;
}
void handle_attributes(const Json::Value& section,Json::Value& course_term) {
// Need to do this because for some reason attributes are by
// section instead of by course
void handle_attributes(const Json::Value& section,
Json::Value& course_term) {
// This mess is basically C++'s string split but not using
// as much memory as an actual string split
const auto delim = std::regex(" and |, ");
@ -144,25 +220,22 @@ void handle_attributes(const Json::Value& section,Json::Value& course_term) {
// Makes the JSON list of attributes
while(attributes_itr != end_itr
&& !attributes_itr->str().empty()) {
handle_attribute(attributes_itr->str(),course_term);
handle_attribute(attributes_itr->str(),course_term["attributes"]);
attributes_itr++;
}
}
void handle_attribute(const std::string& attribute,Json::Value& course_term) {
std::string attr;
void handle_attribute(const std::string& attribute,
Json::Value& attributes) {
if(attribute == "Communication Intensive") {
attr = "[CI]";
attributes.append("[CI]");
} else if(attribute == "Writing Intensive") {
attr = "[WI]";
attributes.append("[WI]");
} else if(attribute == "HASS Inquiry") {
attr = "[HInq]";
attributes.append("[HInq]");
} else if(attribute == "Culminating Exp/Capstone") {
attr = "[CulmExp]";
attributes.append("[CulmExp]");
} else if(attribute == "PDII Option for Engr Majors") {
attr = "[PDII]";
}
if(!attr.empty()) {
course_term["attributes"].append(attr);
attributes.append("[PDII]");
}
}