diff --git a/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.cpp b/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.cpp index 5e0ddd85..2e1d60e2 100644 --- a/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.cpp +++ b/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.cpp @@ -1,4 +1,4 @@ -ConnectedRoute::ConnectedRoute(std::string &line, HighwaySystem *sys, ErrorList &el, std::list &route_list) +ConnectedRoute::ConnectedRoute(std::string &line, HighwaySystem *sys, ErrorList &el) { mileage = 0; // parse chopped routes csv line @@ -29,19 +29,28 @@ ConnectedRoute::ConnectedRoute(std::string &line, HighwaySystem *sys, ErrorList el.add_error("groupname > " + std::to_string(DBFieldLength::city) + " bytes in " + system->systemname + "_con.csv line: " + line); // roots + lower(roots_str.data()); int rootOrder = 0; size_t l = 0; for (size_t r = 0; r != -1; l = r+1) { r = roots_str.find(',', l); - Route *root = route_by_root(roots_str.substr(l, r-l), route_list); - if (!root) el.add_error("Could not find Route matching ConnectedRoute root " + roots_str.substr(l, r-l) + - " in system " + system->systemname + '.'); - else { roots.push_back(root); + try { Route *root = Route::root_hash.at(roots_str.substr(l, r-l)); + roots.push_back(root); + if (root->con_route) + el.add_error("Duplicate root in " + sys->systemname + "_con.csv: " + root->root + + " already in " + root->con_route->system->systemname + "_con.csv"); + if (system != root->system) + el.add_error("System mismatch: chopped route " + root->root + " from " + root->system->systemname + + ".csv in connected route in " + system->systemname + "_con.csv"); root->con_route = this; // save order of route in connected route root->rootOrder = rootOrder; - } - rootOrder++; + rootOrder++; + } + catch (std::out_of_range& oor) + { el.add_error("Could not find Route matching ConnectedRoute root " + roots_str.substr(l, r-l) + + " in system " + system->systemname + '.'); + } } if (roots.size() < 1) el.add_error("No roots in " + system->systemname + "_con.csv line: " + line); } @@ -72,7 +81,7 @@ std::string ConnectedRoute::readable_name() return ans; } -std::string ConnectedRoute::list_lines(int pos, int len, std::string newline, size_t indent) +/*std::string ConnectedRoute::list_lines(int pos, int len, std::string newline, size_t indent) { // return .list file lines marking (len) consecutive // segments, starting at waypoint (pos) segments into route //std::cout << "\nDEBUG: list_lines for " << readable_name() << " (" << roots.size() << " connected root(s))" << std::endl; @@ -86,4 +95,4 @@ std::string ConnectedRoute::list_lines(int pos, int len, std::string newline, si // strip final newline while (lines.back() == '\n' || lines.back() == '\r') lines.pop_back(); return lines; -} +}//*/ diff --git a/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.h b/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.h index c9d05cfe..cf75dba6 100644 --- a/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.h +++ b/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.h @@ -12,10 +12,10 @@ class ConnectedRoute double mileage; // will be computed for routes in active & preview systems - ConnectedRoute(std::string &, HighwaySystem *, ErrorList &, std::list &); + ConnectedRoute(std::string &, HighwaySystem *, ErrorList &); std::string connected_rtes_line(); std::string csv_line(); std::string readable_name(); - std::string list_lines(int, int, std::string, size_t); + //std::string list_lines(int, int, std::string, size_t); }; diff --git a/siteupdate/cplusplus/classes/DatacheckEntry.cpp b/siteupdate/cplusplus/classes/DatacheckEntry.cpp index 8aa0ec0d..0530a3f3 100644 --- a/siteupdate/cplusplus/classes/DatacheckEntry.cpp +++ b/siteupdate/cplusplus/classes/DatacheckEntry.cpp @@ -3,19 +3,21 @@ class DatacheckEntry route is a pointer to the route with a datacheck error - labels is a list of labels that are related to the error (such - as the endpoints of a too-long segment or the three points that - form a sharp angle) + label1, label2 & label3 are labels that are related to the error + (such as the endpoints of a too-long segment or the three points + that form a sharp angle) code is the error code | info is additional string, one of: | information, if used: -----------------------+-------------------------------------------- BAD_ANGLE | BUS_WITH_I | + DISCONNECTED_ROUTE | adjacent root's expected connection point DUPLICATE_COORDS | coordinate pair DUPLICATE_LABEL | HIDDEN_JUNCTION | number of incident edges in TM master graph HIDDEN_TERMINUS | + INTERSTATE_NO_HYPHEN | INVALID_FINAL_CHAR | final character in label INVALID_FIRST_CHAR | first character in label other than * LABEL_INVALID_CHAR | @@ -34,7 +36,7 @@ class DatacheckEntry NONTERMINAL_UNDERSCORE | OUT_OF_BOUNDS | coordinate pair SHARP_ANGLE | angle in degrees - US_BANNER | + US_LETTER | VISIBLE_DISTANCE | distance in miles VISIBLE_HIDDEN_COLOC | hidden point at same coordinates diff --git a/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.cpp b/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.cpp index 579d8e81..24158034 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.cpp +++ b/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.cpp @@ -1,8 +1,3 @@ -// constants for more human-readable format masks -const unsigned char HGEdge::simple = 1; -const unsigned char HGEdge::collapsed = 2; -const unsigned char HGEdge::traveled = 4; - HGEdge::HGEdge(HighwaySegment *s, HighwayGraph *graph) { // initial construction is based on a HighwaySegment s_written = 0; // simple diff --git a/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.h b/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.h index 7b6374e0..3f29ba3a 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.h +++ b/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.h @@ -12,9 +12,9 @@ class HGEdge unsigned char format; // constants for more human-readable format masks - static const unsigned char simple; - static const unsigned char collapsed; - static const unsigned char traveled; + static const unsigned char simple = 1; + static const unsigned char collapsed = 2; + static const unsigned char traveled = 4; HGEdge(HighwaySegment *, HighwayGraph *); HGEdge(HGVertex *, unsigned char); diff --git a/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp b/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp index 8ae6bfaf..238d9d6b 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp +++ b/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp @@ -96,7 +96,7 @@ class HighwayGraph // compress edges adjacent to hidden vertices counter = 0; std::cout << et.et() + "Compressing collapsed edges" << std::flush; - for (std::pair wv : vertices) + for (std::pair& wv : vertices) { if (counter % 10000 == 0) std::cout << '.' << std::flush; counter++; if (!wv.second->visibility) @@ -146,7 +146,7 @@ class HighwayGraph } // end ctor void clear() - { for (std::pair wv : vertices) delete wv.second; + { for (std::pair& wv : vertices) delete wv.second; vertex_names.clear(); waypoint_naming_log.clear(); vertices.clear(); diff --git a/siteupdate/cplusplus/classes/GraphGeneration/PlaceRadius.cpp b/siteupdate/cplusplus/classes/GraphGeneration/PlaceRadius.cpp index 63a4a77e..78b522a3 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/PlaceRadius.cpp +++ b/siteupdate/cplusplus/classes/GraphGeneration/PlaceRadius.cpp @@ -10,10 +10,10 @@ bool PlaceRadius::contains_vertex(HGVertex *v) {return contains_vertex(v->lat, v bool PlaceRadius::contains_vertex(double vlat, double vlng) { /* return whether coordinates are within this area */ // convert to radians to compute distance - double rlat1 = lat * (Waypoint::pi/180); - double rlng1 = lng * (Waypoint::pi/180); - double rlat2 = vlat * (Waypoint::pi/180); - double rlng2 = vlng * (Waypoint::pi/180); + double rlat1 = lat * (pi/180); + double rlng1 = lng * (pi/180); + double rlat2 = vlat * (pi/180); + double rlng2 = vlng * (pi/180); /* original formula double ans = acos(cos(rlat1)*cos(rlng1)*cos(rlat2)*cos(rlng2) +\ @@ -48,10 +48,10 @@ std::unordered_set PlaceRadius::vertices(WaypointQuadtree *qt, Highwa // N/S sanity check: If lat is <= r/2 miles to the N or S pole, lngdelta calculation will fail. // In these cases, our place radius will span the entire "width" of the world, from -180 to +180 degrees. - if (90-fabs(lat)*(Waypoint::pi/180) <= r/7926.2) return v_search(qt, g, -180, +180); + if (90-fabs(lat)*(pi/180) <= r/7926.2) return v_search(qt, g, -180, +180); // width, in degrees longitude, of our bounding box for quadtree search - double lngdelta = acos((cos(r/3963.1) - pow(sin(lat*(Waypoint::pi/180)),2)) / pow(cos(lat*(Waypoint::pi/180)),2)) / (Waypoint::pi/180); + double lngdelta = acos((cos(r/3963.1) - pow(sin(lat*(pi/180)),2)) / pow(cos(lat*(pi/180)),2)) / (pi/180); double w_bound = lng-lngdelta; double e_bound = lng+lngdelta; @@ -92,8 +92,8 @@ std::unordered_set PlaceRadius::v_search(WaypointQuadtree *qt, Highwa // if we're not a terminal quadrant, we need to determine which // of our child quadrants we need to search and recurse into each else { //printf("DEBUG: recursive case, mid_lat=%.17g mid_lng=%.17g\n", qt->mid_lat, qt->mid_lng); fflush(stdout); - bool look_n = (lat + r/3963.1/(Waypoint::pi/180)) >= qt->mid_lat; - bool look_s = (lat - r/3963.1/(Waypoint::pi/180)) <= qt->mid_lat; + bool look_n = (lat + r/3963.1/(pi/180)) >= qt->mid_lat; + bool look_s = (lat - r/3963.1/(pi/180)) <= qt->mid_lat; bool look_e = e_bound >= qt->mid_lng; bool look_w = w_bound <= qt->mid_lng; //std::cout << "DEBUG: recursive case, " << look_n << " " << look_s << " " << look_e << " " << look_w << std::endl; diff --git a/siteupdate/cplusplus/classes/HighwaySegment/HighwaySegment.cpp b/siteupdate/cplusplus/classes/HighwaySegment/HighwaySegment.cpp index 30cccdf1..4b5d009f 100644 --- a/siteupdate/cplusplus/classes/HighwaySegment/HighwaySegment.cpp +++ b/siteupdate/cplusplus/classes/HighwaySegment/HighwaySegment.cpp @@ -49,19 +49,15 @@ unsigned int HighwaySegment::index() /*std::string HighwaySegment::concurrent_travelers_sanity_check() { if (route->system->devel()) return ""; if (concurrent) - for (HighwaySegment *conc : *concurrent) - { if (clinched_by.size() != conc->clinched_by.size()) - { if (conc->route->system->devel()) continue; + for (HighwaySegment *other : *concurrent) + { if (clinched_by.size() != other->clinched_by.size()) + { if (other->route->system->devel()) continue; return "[" + str() + "] clinched by " + std::to_string(clinched_by.size()) + " travelers; [" \ - + conc->str() + "] clinched by " + std::to_string(conc->clinched_by.size()) + '\n'; + + other->str() + "] clinched by " + std::to_string(other->clinched_by.size()) + '\n'; } else for (TravelerList *t : clinched_by) - { std::list::iterator ct; - for (ct = conc->clinched_by.begin(); ct != conc->clinched_by.end(); ct++) - if (*ct == t) break; - if (ct == conc->clinched_by.end()) - return t->traveler_name + " has clinched [" + str() + "], but not [" + conc->str() + "]\n"; - } + if (other->clinched_by.find(t) == other->clinched_by.end()) + return t->traveler_name + " has clinched [" + str() + "], but not [" + other->str() + "]\n"; } return ""; }//*/ diff --git a/siteupdate/cplusplus/classes/HighwaySystem.cpp b/siteupdate/cplusplus/classes/HighwaySystem.cpp index fbdd9054..95dca4ca 100644 --- a/siteupdate/cplusplus/classes/HighwaySystem.cpp +++ b/siteupdate/cplusplus/classes/HighwaySystem.cpp @@ -1,4 +1,3 @@ -//FIXME try to break strtok. That goes for all strtok project-wide. class HighwaySystem { /* This class encapsulates the contents of one .csv file that represents the collection of highways within a system. @@ -28,6 +27,8 @@ class HighwaySystem std::list con_route_list; std::unordered_map mileage_by_region; std::unordered_set vertices; + std::unordered_setlistnamesinuse, unusedaltroutenames; + std::mutex lniu_mtx, uarn_mtx; bool is_valid; HighwaySystem(std::string &line, ErrorList &el, std::string path, std::string &systemsfile, @@ -74,12 +75,15 @@ class HighwaySystem if (level_str != "active" && level_str != "preview" && level_str != "devel") el.add_error("Unrecognized level in " + systemsfile + " line: " + line); + std::cout << systemname << '.' << std::flush; + // read chopped routes CSV file.open(path+"/"+systemname+".csv"); if (!file) el.add_error("Could not open "+path+"/"+systemname+".csv"); else { getline(file, line); // ignore header line while(getline(file, line)) - { if (line.back() == 0x0D) line.erase(line.end()-1); // trim DOS newlines + { // trim DOS newlines & trailing whitespace + while ( strchr("\r\t ", line.back()) ) line.pop_back(); if (line.empty()) continue; route_list.emplace_back(line, this, el, region_hash); if (route_list.back().root.empty()) @@ -95,9 +99,10 @@ class HighwaySystem if (!file) el.add_error("Could not open "+path+"/"+systemname+"_con.csv"); else { getline(file, line); // ignore header line while(getline(file, line)) - { if (line.back() == 0x0D) line.erase(line.end()-1); // trim DOS newlines + { // trim DOS newlines & trailing whitespace + while ( strchr("\r\t ", line.back()) ) line.pop_back(); if (line.empty()) continue; - con_route_list.emplace_back(line, this, el, route_list); + con_route_list.emplace_back(line, this, el); } } file.close(); @@ -126,7 +131,7 @@ class HighwaySystem /* Return total system mileage across all regions */ double total_mileage() { double mi = 0; - for (std::pair rm : mileage_by_region) mi += rm.second; + for (std::pair& rm : mileage_by_region) mi += rm.second; return mi; } diff --git a/siteupdate/cplusplus/classes/Region.cpp b/siteupdate/cplusplus/classes/Region.cpp index 4716aa2e..d7daa493 100644 --- a/siteupdate/cplusplus/classes/Region.cpp +++ b/siteupdate/cplusplus/classes/Region.cpp @@ -1,6 +1,6 @@ std::pair *country_or_continent_by_code(std::string code, std::vector> &pair_vector) -{ for (std::vector>::iterator c = pair_vector.begin(); c != pair_vector.end(); c++) - if (c->first == code) return &*c; +{ for (std::pair& c : pair_vector) + if (c.first == code) return &c; return 0; } diff --git a/siteupdate/cplusplus/classes/Route/Route.cpp b/siteupdate/cplusplus/classes/Route/Route.cpp index 6c104d27..609de387 100644 --- a/siteupdate/cplusplus/classes/Route/Route.cpp +++ b/siteupdate/cplusplus/classes/Route/Route.cpp @@ -1,3 +1,4 @@ +std::unordered_map Route::root_hash, Route::pri_list_hash, Route::alt_list_hash; std::mutex Route::awf_mtx; Route::Route(std::string &line, HighwaySystem *sys, ErrorList &el, std::unordered_map ®ion_hash) @@ -7,6 +8,7 @@ Route::Route(std::string &line, HighwaySystem *sys, ErrorList &el, std::unordere mileage = 0; rootOrder = -1; // order within connected route region = 0; // if this stays 0, setup has failed due to bad .csv data + is_reversed = 0; // parse chopped routes csv line size_t NumFields = 8; @@ -52,11 +54,38 @@ Route::Route(std::string &line, HighwaySystem *sys, ErrorList &el, std::unordere if (root.size() > DBFieldLength::root) el.add_error("Root > " + std::to_string(DBFieldLength::root) + " bytes in " + system->systemname + ".csv line: " + line); + lower(root.data()); // alt_route_names - size_t l = 0; - for (size_t r = 0; r != -1; l = r+1) - { r = arn_str.find(',', l); - alt_route_names.emplace_back(arn_str, l, r-l); + upper(arn_str.data()); + size_t len; + for (size_t pos = 0; pos < arn_str.size(); pos += len+1) + { len = strcspn(arn_str.data()+pos, ","); + alt_route_names.emplace_back(arn_str, pos, len); + } + + // insert into root_hash, checking for duplicate root entries + if (!root_hash.insert(std::pair(root, this)).second) + el.add_error("Duplicate root in " + system->systemname + ".csv: " + root + + " already in " + root_hash.at(root)->system->systemname + ".csv"); + // insert list name into pri_list_hash, checking for duplicate .list names + std::string list_name = upper(readable_name()); + if (alt_list_hash.find(list_name) != alt_list_hash.end()) + el.add_error("Duplicate main list name in " + root + ": '" + readable_name() + + "' already points to " + alt_list_hash.at(list_name)->root); + else if (!pri_list_hash.insert(std::pair(list_name, this)).second) + el.add_error("Duplicate main list name in " + root + ": '" + readable_name() + + "' already points to " + pri_list_hash.at(list_name)->root); + // insert alt names into alt_list_hash, checking for duplicate .list names + for (std::string& a : alt_route_names) + { list_name = upper(rg_str + ' ' + a); + if (pri_list_hash.find(list_name) != pri_list_hash.end()) + el.add_error("Duplicate alt route name in " + root + ": '" + region->code + ' ' + a + + "' already points to " + pri_list_hash.at(list_name)->root); + else if (!alt_list_hash.insert(std::pair(list_name, this)).second) + el.add_error("Duplicate alt route name in " + root + ": '" + region->code + ' ' + a + + "' already points to " + alt_list_hash.at(list_name)->root); + // populate unused set + system->unusedaltroutenames.insert(list_name); } } @@ -132,16 +161,16 @@ double Route::clinched_by_traveler(TravelerList *t) return miles; } -std::string Route::list_line(int beg, int end) +/*std::string Route::list_line(int beg, int end) { /* Return a .list file line from (beg) to (end), these being indices to the point_list vector. These values can be "out-of-bounds" when getting lines for connected routes. If so, truncate or return "". */ - if (beg >= int(point_list.size()) || end <= 0) return ""; +/* if (beg >= int(point_list.size()) || end <= 0) return ""; if (end >= int(point_list.size())) end = point_list.size()-1; if (beg < 0) beg = 0; return readable_name() + " " + point_list[beg]->label + " " + point_list[end]->label; -} +}//*/ void Route::write_nmp_merged(std::string filename) { mkdir(filename.data(), 0777); @@ -180,8 +209,19 @@ void Route::write_nmp_merged(std::string filename) wptfile.close(); } -Route *route_by_root(std::string root, std::list &route_list) -{ for (std::list::iterator r = route_list.begin(); r != route_list.end(); r++) - if (r->root == root) return &*r; - return 0; +inline void Route::store_traveled_segments(TravelerList* t, unsigned int beg, unsigned int end) +{ // store clinched segments with traveler and traveler with segments + for (unsigned int pos = beg; pos < end; pos++) + { HighwaySegment *hs = segment_list[pos]; + hs->add_clinched_by(t); + t->clinched_segments.insert(hs); + } +} + +inline Waypoint* Route::con_beg() +{ return is_reversed ? point_list.back() : point_list.front(); +} + +inline Waypoint* Route::con_end() +{ return is_reversed ? point_list.front() : point_list.back(); } diff --git a/siteupdate/cplusplus/classes/Route/Route.h b/siteupdate/cplusplus/classes/Route/Route.h index b1949da7..3f6a4a8d 100644 --- a/siteupdate/cplusplus/classes/Route/Route.h +++ b/siteupdate/cplusplus/classes/Route/Route.h @@ -51,17 +51,21 @@ class Route std::vector point_list; std::unordered_set labels_in_use; std::unordered_set unused_alt_labels; + std::unordered_set duplicate_labels; + std::unordered_map pri_label_hash, alt_label_hash; + static std::unordered_map root_hash, pri_list_hash, alt_list_hash; static std::mutex awf_mtx; // for locking the all_wpt_files set when erasing processed WPTs std::mutex liu_mtx; // for locking the labels_in_use set when inserting labels during TravelerList processing std::mutex ual_mtx; // for locking the unused_alt_labels set when removing in-use alt_labels std::vector segment_list; double mileage; int rootOrder; + bool is_reversed; Route(std::string &, HighwaySystem *, ErrorList &, std::unordered_map &); std::string str(); - void read_wpt(WaypointQuadtree *, ErrorList *, std::string, std::mutex *, DatacheckEntryList *, std::unordered_set *); + void read_wpt(WaypointQuadtree *, ErrorList *, std::string, bool, DatacheckEntryList *, std::unordered_set *); void print_route(); HighwaySegment* find_segment_by_waypoints(Waypoint*, Waypoint*); std::string chopped_rtes_line(); @@ -70,6 +74,9 @@ class Route std::string list_entry_name(); std::string name_no_abbrev(); double clinched_by_traveler(TravelerList *); - std::string list_line(int, int); + //std::string list_line(int, int); void write_nmp_merged(std::string); + inline void store_traveled_segments(TravelerList*, unsigned int, unsigned int); + inline Waypoint* con_beg(); + inline Waypoint* con_end(); }; diff --git a/siteupdate/cplusplus/classes/Route/read_wpt.cpp b/siteupdate/cplusplus/classes/Route/read_wpt.cpp index 1be5cadc..4377c79b 100644 --- a/siteupdate/cplusplus/classes/Route/read_wpt.cpp +++ b/siteupdate/cplusplus/classes/Route/read_wpt.cpp @@ -1,5 +1,5 @@ void Route::read_wpt -( WaypointQuadtree *all_waypoints, ErrorList *el, std::string path, std::mutex *strtok_mtx, +( WaypointQuadtree *all_waypoints, ErrorList *el, std::string path, bool usa_flag, DatacheckEntryList *datacheckerrors, std::unordered_set *all_wpt_files ) { /* read data into the Route's waypoint list from a .wpt file */ @@ -12,7 +12,7 @@ void Route::read_wpt std::vector lines; std::ifstream file(filename); if (!file) - { el->add_error("Could not open " + filename); + { el->add_error("[Errno 2] No such file or directory: '" + filename + '\''); file.close(); return; } @@ -23,12 +23,15 @@ void Route::read_wpt file.read(wptdata, wptdatasize); wptdata[wptdatasize] = 0; // add null terminator file.close(); - strtok_mtx->lock(); - for (char *token = strtok(wptdata, "\r\n"); token; token = strtok(0, "\r\n") ) lines.emplace_back(token); - strtok_mtx->unlock(); + + // split file into lines + size_t spn = 0; + for (char* c = wptdata; *c; c += spn) + { for (spn = strcspn(c, "\n\r"); c[spn] == '\n' || c[spn] == '\r'; spn++) c[spn] = 0; + lines.emplace_back(c); + } + lines.push_back(wptdata+wptdatasize+1); // add a dummy "past-the-end" element to make lines[l+1]-2 work - // set to be used per-route to find label duplicates - std::unordered_set all_route_labels; // set to be used for finding duplicate coordinates std::unordered_set coords_used; double vis_dist = 0; @@ -38,14 +41,14 @@ void Route::read_wpt for (unsigned int l = 0; l < lines.size()-1; l++) { // strip whitespace while (lines[l][0] == ' ' || lines[l][0] == '\t') lines[l]++; - char * endchar = lines[l+1]-2; // -2 skips over the 0 inserted by strtok - if (*endchar == 0) endchar--; // skip back one more for CRLF cases FIXME what about lines followed by blank lines? + char * endchar = lines[l+1]-2; // -2 skips over the 0 inserted while splitting wptdata into lines + while (*endchar == 0) endchar--; // skip back more for CRLF cases, and lines followed by blank lines while (*endchar == ' ' || *endchar == '\t') { *endchar = 0; endchar--; } if (lines[l][0] == 0) continue; - Waypoint *w = new Waypoint(lines[l], this, strtok_mtx, datacheckerrors); + Waypoint *w = new Waypoint(lines[l], this, datacheckerrors); // deleted on termination of program, or immediately below if invalid bool malformed_url = w->lat == 0 && w->lng == 0; bool label_too_long = w->label_too_long(datacheckerrors); @@ -54,22 +57,12 @@ void Route::read_wpt continue; } point_list.push_back(w); - // populate unused alt labels - for (size_t i = 0; i < w->alt_labels.size(); i++) - { std::string al = w->alt_labels[i]; - // strip out leading '+' - while (al[0] == '+') al.erase(0, 1); //TODO would erase via iterator be any faster? - // convert to upper case - for (size_t c = 0; c < al.size(); c++) - if (al[c] >= 'a' && al[c] <= 'z') al[c] -= 32; - unused_alt_labels.insert(al); - } all_waypoints->insert(w, 1); // single-point Datachecks, and HighwaySegment w->out_of_bounds(datacheckerrors, fstr); - w->duplicate_label(datacheckerrors, all_route_labels); w->duplicate_coords(datacheckerrors, coords_used, fstr); + w->label_invalid_char(datacheckerrors); if (point_list.size() > 1) { w->distance_update(datacheckerrors, fstr, vis_dist, point_list[point_list.size()-2]); // add HighwaySegment, if not first point @@ -78,17 +71,20 @@ void Route::read_wpt } // checks for visible points if (!w->is_hidden) - { w->visible_distance(datacheckerrors, fstr, vis_dist, last_visible); - const char *slash = strchr(w->label.data(), '/'); + { const char *slash = strchr(w->label.data(), '/'); + if (usa_flag && w->label.size() >= 2) + { w->bus_with_i(datacheckerrors); + w->interstate_no_hyphen(datacheckerrors); + w->us_letter(datacheckerrors); + } + w->label_invalid_ends(datacheckerrors); + w->label_looks_hidden(datacheckerrors); + w->label_parens(datacheckerrors); w->label_selfref(datacheckerrors, slash); w->label_slashes(datacheckerrors, slash); + w->lacks_generic(datacheckerrors); w->underscore_datachecks(datacheckerrors, slash); - w->label_parens(datacheckerrors); - w->label_invalid_char(datacheckerrors, w->label); - for (std::string &a : w->alt_labels) w->label_invalid_char(datacheckerrors, a); - w->label_invalid_ends(datacheckerrors); - w->bus_with_i(datacheckerrors); - w->label_looks_hidden(datacheckerrors); + w->visible_distance(datacheckerrors, fstr, vis_dist, last_visible); } } delete[] wptdata; diff --git a/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp b/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp index e6419a35..6598965e 100644 --- a/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp +++ b/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp @@ -1,234 +1,129 @@ -class TravelerList -{ /* This class encapsulates the contents of one .list file - that represents the travels of one individual user. +TravelerList::TravelerList(std::string travname, ErrorList *el, Arguments *args) +{ active_systems_traveled = 0; + active_systems_clinched = 0; + preview_systems_traveled = 0; + preview_systems_clinched = 0; + unsigned int list_entries = 0; + traveler_num = new unsigned int[args->numthreads]; + // deleted on termination of program + traveler_name = travname.substr(0, travname.size()-5); // strip ".list" from end of travname + if (traveler_name.size() > DBFieldLength::traveler) + el->add_error("Traveler name " + traveler_name + " > " + std::to_string(DBFieldLength::traveler) + "bytes"); + std::ofstream log(args->logfilepath+"/users/"+traveler_name+".log"); + std::ofstream splist; + if (args->splitregionpath != "") splist.open(args->splitregionpath+"/list_files/"+travname); + time_t StartTime = time(0); + log << "Log file created at: "; + alltrav_mtx.lock(); + log << ctime(&StartTime); + alltrav_mtx.unlock(); + std::vector lines; + std::vector endlines; + std::ifstream file(args->userlistfilepath+"/"+travname); + // we can't getline here because it only allows one delimiter, and we need two; '\r' and '\n'. + // at least one .list file contains newlines using only '\r' (0x0D): + // https://github.com/TravelMapping/UserData/blob/6309036c44102eb3325d49515b32c5eef3b3cb1e/list_files/whopperman.list + file.seekg(0, std::ios::end); + unsigned long listdatasize = file.tellg(); + file.seekg(0, std::ios::beg); + char *listdata = new char[listdatasize+1]; + file.read(listdata, listdatasize); + listdata[listdatasize] = 0; // add null terminator + file.close(); - A list file consists of lines of 4 values: - region route_name start_waypoint end_waypoint + // get canonical newline for writing splitregion .list files + std::string newline; + unsigned long c = 0; + while (listdata[c] != '\r' && listdata[c] != '\n' && c < listdatasize) c++; + if (listdata[c] == '\r') + if (listdata[c+1] == '\n') newline = "\r\n"; + else newline = "\r"; + else if (listdata[c] == '\n') newline = "\n"; + // Use CRLF as failsafe if .list file contains no newlines. + else newline = "\r\n"; - which indicates that the user has traveled the highway names - route_name in the given region between the waypoints named - start_waypoint end_waypoint - */ - public: - std::mutex ap_mi_mtx, ao_mi_mtx, sr_mi_mtx; - std::unordered_set clinched_segments; - std::string traveler_name; - std::unordered_map active_preview_mileage_by_region; // total mileage per region, active+preview only - std::unordered_map active_only_mileage_by_region; // total mileage per region, active only - std::unordered_map> system_region_mileages; // mileage per region per system - std::unordered_map> con_routes_traveled; // mileage per ConRte per system - // TODO is this necessary? - // ConRtes by definition exist in one system only - std::unordered_map routes_traveled; // mileage per traveled route - std::unordered_map con_routes_clinched; // clinch count per system - //std::unordered_map routes_clinched; // commented out in original siteupdate.py - unsigned int *traveler_num; - unsigned int active_systems_traveled; - unsigned int active_systems_clinched; - unsigned int preview_systems_traveled; - unsigned int preview_systems_clinched; - static std::mutex alltrav_mtx; // for locking the traveler_lists list when reading .lists from disk - - TravelerList(std::string travname, std::unordered_map *route_hash, ErrorList *el, Arguments *args, std::mutex *strtok_mtx) - { active_systems_traveled = 0; - active_systems_clinched = 0; - preview_systems_traveled = 0; - preview_systems_clinched = 0; - unsigned int list_entries = 0; - traveler_num = new unsigned int[args->numthreads]; - // deleted on termination of program - traveler_name = travname.substr(0, travname.size()-5); // strip ".list" from end of travname - if (traveler_name.size() > DBFieldLength::traveler) - el->add_error("Traveler name " + traveler_name + " > " + std::to_string(DBFieldLength::traveler) + "bytes"); - std::ofstream log(args->logfilepath+"/users/"+traveler_name+".log"); - std::ofstream splist; - if (args->splitregionpath != "") splist.open(args->splitregionpath+"/list_files/"+travname); - time_t StartTime = time(0); - log << "Log file created at: " << ctime(&StartTime); - std::vector lines; - std::vector endlines; - std::ifstream file(args->userlistfilepath+"/"+travname); - // we can't getline here because it only allows one delimiter, and we need two; '\r' and '\n'. - // at least one .list file contains newlines using only '\r' (0x0D): - // https://github.com/TravelMapping/UserData/blob/6309036c44102eb3325d49515b32c5eef3b3cb1e/list_files/whopperman.list - file.seekg(0, std::ios::end); - unsigned long listdatasize = file.tellg(); - file.seekg(0, std::ios::beg); - char *listdata = new char[listdatasize+1]; - file.read(listdata, listdatasize); - listdata[listdatasize] = 0; // add null terminator - file.close(); - - // get canonical newline for writing splitregion .list files - std::string newline; - unsigned long c = 0; - while (listdata[c] != '\r' && listdata[c] != '\n' && c < listdatasize) c++; - if (listdata[c] == '\r') - if (listdata[c+1] == '\n') newline = "\r\n"; - else newline = "\r"; - else if (listdata[c] == '\n') newline = "\n"; - // Use CRLF as failsafe if .list file contains no newlines. - else newline = "\r\n"; - - // separate listdata into series of lines & newlines - size_t spn = 0; - for (char *c = listdata; *c; c += spn) - { endlines.push_back(""); - spn = strcspn(c, "\r\n"); - while (c[spn] == '\r' || c[spn] == '\n') - { endlines.back().push_back(c[spn]); - c[spn] = 0; - spn++; - } - lines.push_back(c); + // separate listdata into series of lines & newlines + size_t spn = 0; + for (char *c = listdata; *c; c += spn) + { endlines.push_back(""); + for (spn = strcspn(c, "\n\r"); c[spn] == '\n' || c[spn] == '\r'; spn++) + { endlines.back().push_back(c[spn]); + c[spn] = 0; } - lines.push_back(listdata+listdatasize+1); // add a dummy "past-the-end" element to make lines[l+1]-2 work - // strip UTF-8 byte order mark if present - if (!strncmp(lines[0], "\xEF\xBB\xBF", 3)) lines[0] += 3; - - for (unsigned int l = 0; l < lines.size()-1; l++) - { std::string orig_line(lines[l]); - // strip whitespace - while (lines[l][0] == ' ' || lines[l][0] == '\t') lines[l]++; - char * endchar = lines[l+1]-2; // -2 skips over the 0 inserted while separating listdata into lines - while (*endchar == 0) endchar--; // skip back more for CRLF cases, and lines followed by blank lines - while (*endchar == ' ' || *endchar == '\t') - { *endchar = 0; - endchar--; - } - std::string trim_line(lines[l]); - // ignore empty or "comment" lines - if (lines[l][0] == 0 || lines[l][0] == '#') - { splist << orig_line << endlines[l]; - continue; - } - // process fields in line - std::vector fields; - strtok_mtx->lock(); - for (char *token = strtok(lines[l], " \t"); token; token = strtok(0, " \t") ) fields.push_back(token); - strtok_mtx->unlock(); - if (fields.size() != 4) - // OK if 5th field exists and starts with # - if (fields.size() < 5 || fields[4][0] != '#') - { log << "Incorrect format line: " << trim_line << '\n'; - splist << orig_line << endlines[l]; - continue; - } + lines.push_back(c); + } + lines.push_back(listdata+listdatasize+1); // add a dummy "past-the-end" element to make lines[l+1]-2 work + // strip UTF-8 byte order mark if present + if (!strncmp(lines[0], "\xEF\xBB\xBF", 3)) + { lines[0] += 3; + splist << "\xEF\xBB\xBF"; + } - // find the root that matches in some system and when we do, match labels - std::string route_entry = lower(std::string(fields[1])); - std::string lookup = std::string(lower(fields[0])) + " " + route_entry; - try { Route *r = route_hash->at(lookup); - for (std:: string a : r->alt_route_names) - if (route_entry == lower(a)) - { log << "Note: deprecated route name " << fields[1] - << " -> canonical name " << r->list_entry_name() << " in line " << trim_line << '\n'; - break; - } - if (r->system->devel()) - { log << "Ignoring line matching highway in system in development: " << trim_line << '\n'; - splist << orig_line << endlines[l]; - continue; - } - // r is a route match, r.root is our root, and we need to find - // canonical waypoint labels, ignoring case and leading - // "+" or "*" when matching - std::vector point_indices; - unsigned int checking_index = 0; - for (Waypoint *w : r->point_list) - { std::string lower_label = lower(w->label); - while (*lower(fields[2]) == '*' || *fields[2] == '+') fields[2]++; - while (*lower(fields[3]) == '*' || *fields[3] == '+') fields[3]++; - while (lower_label.front() == '+' || lower_label.front() == '*') lower_label.erase(lower_label.begin()); - if (fields[2] == lower_label || fields[3] == lower_label) - { point_indices.push_back(checking_index); - r->liu_mtx.lock(); - r->labels_in_use.insert(upper(lower_label)); - r->liu_mtx.unlock(); - } - else { for (std::string &alt : w->alt_labels) - { lower_label = lower(alt); - while (lower_label.front() == '+') lower_label.erase(lower_label.begin()); - if (fields[2] == lower_label || fields[3] == lower_label) - { point_indices.push_back(checking_index); - r->liu_mtx.lock(); - r->labels_in_use.insert(upper(lower_label)); - r->liu_mtx.unlock(); - // if we have not yet used this alt label, remove it from the unused set - r->ual_mtx.lock(); - r->unused_alt_labels.erase(upper(lower_label)); - r->ual_mtx.unlock(); - } - } - } - checking_index++; - } - if (point_indices.size() != 2) - { bool invalid_char = 0; - for (size_t c = 0; c < trim_line.size(); c++) - if (iscntrl(trim_line[c])) - { trim_line[c] = '?'; - invalid_char = 1; - } - log << "Waypoint label(s) not found in line: " << trim_line; - if (invalid_char) log << " [contains invalid character(s)]"; - log << '\n'; - splist << orig_line << endlines[l]; - } - else { list_entries++; - // find the segments we just matched and store this traveler with the - // segments and the segments with the traveler (might not need both - // ultimately) - for (unsigned int wp_pos = point_indices[0]; wp_pos < point_indices[1]; wp_pos++) - { HighwaySegment *hs = r->segment_list[wp_pos]; - hs->add_clinched_by(this); - clinched_segments.insert(hs); - } - #include "splitregion.cpp" - } - } - catch (const std::out_of_range& oor) - { bool invalid_char = 0; - for (size_t c = 0; c < trim_line.size(); c++) - if (iscntrl(trim_line[c])) - { trim_line[c] = '?'; - invalid_char = 1; - } - log << "Unknown region/highway combo in line: " << trim_line; - if (invalid_char) log << " [contains invalid character(s)]"; - log << '\n'; - splist << orig_line << endlines[l]; - } + for (unsigned int l = 0; l < lines.size()-1; l++) + { std::string orig_line(lines[l]); + // strip whitespace + while (lines[l][0] == ' ' || lines[l][0] == '\t') lines[l]++; + char * endchar = lines[l+1]-2; // -2 skips over the 0 inserted while separating listdata into lines + while (*endchar == 0 && endchar > lines[l]) endchar--; // skip back more for CRLF cases, and lines followed by blank lines + while (*endchar == ' ' || *endchar == '\t') + { *endchar = 0; + endchar--; } - delete[] listdata; - log << "Processed " << list_entries << " good lines marking " << clinched_segments.size() << " segments traveled.\n"; - log.close(); - splist.close(); + std::string trim_line(lines[l]); + // ignore empty or "comment" lines + if (lines[l][0] == 0 || lines[l][0] == '#') + { splist << orig_line << endlines[l]; + continue; + } + // process fields in line + std::vector fields; + size_t spn = 0; + for (char* c = lines[l]; *c; c += spn) + { for (spn = strcspn(c, " \t"); c[spn] == ' ' || c[spn] == '\t'; spn++) c[spn] = 0; + if (*c == '#') break; + else fields.push_back(c); + } + if (fields.size() == 4) + { + #include "mark_chopped_route_segments.cpp" + } + else if (fields.size() == 6) + { + #include "mark_connected_route_segments.cpp" + } + else { log << "Incorrect format line (4 or 6 fields expected, found " + << fields.size() << "): " << trim_line << '\n'; + splist << orig_line << endlines[l]; + } } + delete[] listdata; + log << "Processed " << list_entries << " good lines marking " << clinched_segments.size() << " segments traveled.\n"; + log.close(); + splist.close(); +} - /* Return active mileage across all regions */ - double active_only_miles() - { double mi = 0; - for (std::pair rm : active_only_mileage_by_region) mi += rm.second; - return mi; - } +/* Return active mileage across all regions */ +double TravelerList::active_only_miles() +{ double mi = 0; + for (std::pair& rm : active_only_mileage_by_region) mi += rm.second; + return mi; +} - /* Return active+preview mileage across all regions */ - double active_preview_miles() - { double mi = 0; - for (std::pair rm : active_preview_mileage_by_region) mi += rm.second; - return mi; - } +/* Return active+preview mileage across all regions */ +double TravelerList::active_preview_miles() +{ double mi = 0; + for (std::pair& rm : active_preview_mileage_by_region) mi += rm.second; + return mi; +} - /* Return mileage across all regions for a specified system */ - double system_region_miles(HighwaySystem *h) - { double mi = 0; - for (std::pair rm : system_region_mileages.at(h)) mi += rm.second; - return mi; - } +/* Return mileage across all regions for a specified system */ +double TravelerList::system_region_miles(HighwaySystem *h) +{ double mi = 0; + for (std::pair& rm : system_region_mileages.at(h)) mi += rm.second; + return mi; +} - #include "userlog.cpp" -}; +#include "userlog.cpp" std::mutex TravelerList::alltrav_mtx; diff --git a/siteupdate/cplusplus/classes/TravelerList/TravelerList.h b/siteupdate/cplusplus/classes/TravelerList/TravelerList.h new file mode 100644 index 00000000..7710dee9 --- /dev/null +++ b/siteupdate/cplusplus/classes/TravelerList/TravelerList.h @@ -0,0 +1,39 @@ +class TravelerList +{ /* This class encapsulates the contents of one .list file + that represents the travels of one individual user. + + A list file consists of lines of 4 values: + region route_name start_waypoint end_waypoint + + which indicates that the user has traveled the highway names + route_name in the given region between the waypoints named + start_waypoint end_waypoint + */ + public: + std::mutex ap_mi_mtx, ao_mi_mtx, sr_mi_mtx; + std::unordered_set clinched_segments; + std::string traveler_name; + std::unordered_map active_preview_mileage_by_region; // total mileage per region, active+preview only + std::unordered_map active_only_mileage_by_region; // total mileage per region, active only + std::unordered_map> system_region_mileages; // mileage per region per system + std::unordered_map> con_routes_traveled; // mileage per ConRte per system + // TODO is this necessary? + // ConRtes by definition exist in one system only + std::unordered_map routes_traveled; // mileage per traveled route + std::unordered_map con_routes_clinched; // clinch count per system + //std::unordered_map routes_clinched; // commented out in original siteupdate.py + unsigned int *traveler_num; + unsigned int active_systems_traveled; + unsigned int active_systems_clinched; + unsigned int preview_systems_traveled; + unsigned int preview_systems_clinched; + // for locking the traveler_lists list when reading .lists from disk + // and avoiding data races when creating userlog timestamps + static std::mutex alltrav_mtx; + + TravelerList(std::string, ErrorList *, Arguments *); + double active_only_miles(); + double active_preview_miles(); + double system_region_miles(HighwaySystem *); + void userlog(ClinchedDBValues *, const double, const double, std::list*, std::string path); +}; diff --git a/siteupdate/cplusplus/classes/TravelerList/mark_chopped_route_segments.cpp b/siteupdate/cplusplus/classes/TravelerList/mark_chopped_route_segments.cpp new file mode 100644 index 00000000..2be83dc9 --- /dev/null +++ b/siteupdate/cplusplus/classes/TravelerList/mark_chopped_route_segments.cpp @@ -0,0 +1,125 @@ +// find the route that matches and when we do, match labels +std::string lookup = upper(fields[0]) + ( ' ' + upper(std::string(fields[1])) ); // leave fields[1] intact for potential AltRouteName note +// look for region/route combo, first in pri_list_hash +std::unordered_map::iterator rit = Route::pri_list_hash.find(lookup); +// and then if not found, in alt_list_hash +if (rit == Route::pri_list_hash.end()) +{ rit = Route::alt_list_hash.find(lookup); + if (rit == Route::alt_list_hash.end()) + { bool invalid_char = 0; + for (char& c : trim_line) + if (iscntrl(c)) + { c = '?'; + invalid_char = 1; + } + log << "Unknown region/highway combo in line: " << trim_line; + if (invalid_char) log << " [contains invalid character(s)]"; + log << '\n'; + splist << orig_line << endlines[l]; + continue; + } + else log << "Note: deprecated route name " << fields[1] + << " -> canonical name " << rit->second->list_entry_name() << " in line: " << trim_line << '\n'; +} +Route* r = rit->second; +if (r->system->devel()) +{ log << "Ignoring line matching highway in system in development: " << trim_line << '\n'; + splist << orig_line << endlines[l]; + continue; +} +// r is a route match, and we need to find +// waypoint indices, ignoring case and leading +// '+' or '*' when matching +unsigned int index1, index2; +while (*fields[2] == '*' || *fields[2] == '+') fields[2]++; +while (*fields[3] == '*' || *fields[3] == '+') fields[3]++; +upper(fields[2]); +upper(fields[3]); +// look for point indices for labels, first in pri_label_hash +std::unordered_map::iterator lit1 = r->pri_label_hash.find(fields[2]); +std::unordered_map::iterator lit2 = r->pri_label_hash.find(fields[3]); +// and then if not found, in alt_label_hash +if (lit1 == r->pri_label_hash.end()) lit1 = r->alt_label_hash.find(fields[2]); +if (lit2 == r->pri_label_hash.end()) lit2 = r->alt_label_hash.find(fields[3]); +// if we did not find matches for both labels... +if (lit1 == r->alt_label_hash.end() || lit2 == r->alt_label_hash.end()) +{ bool invalid_char = 0; + for (char& c : trim_line) + if (iscntrl(c)) + { c = '?'; + invalid_char = 1; + } + for (char* c = fields[2]; *c; c++) if (iscntrl(*c)) *c = '?'; + for (char* c = fields[3]; *c; c++) if (iscntrl(*c)) *c = '?'; + if (lit1 == lit2) + log << "Waypoint labels " << fields[2] << " and " << fields[3] << " not found in line: " << trim_line; + else { log << "Waypoint label "; + log << (lit1 == r->alt_label_hash.end() ? fields[2] : fields[3]); + log << " not found in line: " << trim_line; + } + if (invalid_char) log << " [contains invalid character(s)]"; + log << '\n'; + splist << orig_line << endlines[l]; + continue; +} +// are either of the labels used duplicates? +char duplicate = 0; +if (r->duplicate_labels.find(fields[2]) != r->duplicate_labels.end()) +{ log << r->region->code << ": duplicate label " << fields[2] << " in " << r->root + << ". Please report this error in the TravelMapping forum" + << ". Unable to parse line: " << trim_line << '\n'; + duplicate = 1; +} +if (r->duplicate_labels.find(fields[3]) != r->duplicate_labels.end()) +{ log << r->region->code << ": duplicate label " << fields[3] << " in " << r->root + << ". Please report this error in the TravelMapping forum" + << ". Unable to parse line: " << trim_line << '\n'; + duplicate = 1; +} +if (duplicate) +{ splist << orig_line << endlines[l]; + continue; +} +// if both labels reference the same waypoint... +if (lit1->second == lit2->second) +{ log << "Equivalent waypoint labels mark zero distance traveled in line: " << trim_line << '\n'; + splist << orig_line << endlines[l]; +} +// otherwise both labels are valid; mark in use & proceed +else { r->system->lniu_mtx.lock(); + r->system->listnamesinuse.insert(lookup); + r->system->lniu_mtx.unlock(); + r->system->uarn_mtx.lock(); + r->system->unusedaltroutenames.erase(lookup); + r->system->uarn_mtx.unlock(); + r->liu_mtx.lock(); + r->labels_in_use.insert(fields[2]); + r->labels_in_use.insert(fields[3]); + r->liu_mtx.unlock(); + r->ual_mtx.lock(); + r->unused_alt_labels.erase(fields[2]); + r->unused_alt_labels.erase(fields[3]); + r->ual_mtx.unlock(); + + list_entries++; + bool reverse = 0; + if (lit1->second <= lit2->second) + { index1 = lit1->second; + index2 = lit2->second; + } + else { index1 = lit2->second; + index2 = lit1->second; + reverse = 1; + } + r->store_traveled_segments(this, index1, index2); + // new .list lines for region split-ups + if (args->splitregion == r->region->code) + { + #define r1 r + #define r2 r + #include "splitregion.cpp" + #undef r1 + #undef r2 + } + else splist << orig_line << endlines[l]; + } diff --git a/siteupdate/cplusplus/classes/TravelerList/mark_connected_route_segments.cpp b/siteupdate/cplusplus/classes/TravelerList/mark_connected_route_segments.cpp new file mode 100644 index 00000000..61f70421 --- /dev/null +++ b/siteupdate/cplusplus/classes/TravelerList/mark_connected_route_segments.cpp @@ -0,0 +1,163 @@ +std::string lookup1 = upper(std::string(fields[0]) + ' ' + fields[1]); // leave fields intact for potential AltRouteName note +std::string lookup2 = upper(std::string(fields[3]) + ' ' + fields[4]); // leave fields intact for potential AltRouteName note +// look for region/route combos, first in pri_list_hash +std::unordered_map::iterator rit1 = Route::pri_list_hash.find(lookup1); +std::unordered_map::iterator rit2 = Route::pri_list_hash.find(lookup2); +// and then if not found, in alt_list_hash +if (rit1 == Route::pri_list_hash.end()) +{ rit1 = Route::alt_list_hash.find(lookup1); + if (rit1 != Route::alt_list_hash.end()) + log << "Note: deprecated route name \"" << fields[0] << ' ' << fields[1] + << "\" -> canonical name \"" << rit1->second->readable_name() << "\" in line: " << trim_line << '\n'; +} +if (rit2 == Route::pri_list_hash.end()) +{ rit2 = Route::alt_list_hash.find(lookup2); + if (rit2 != Route::alt_list_hash.end()) + log << "Note: deprecated route name \"" << fields[3] << ' ' << fields[4] + << "\" -> canonical name \"" << rit2->second->readable_name() << "\" in line: " << trim_line << '\n'; +} +if (rit1 == Route::alt_list_hash.end() || rit2 == Route::alt_list_hash.end()) +{ bool invalid_char = 0; + for (char& c : trim_line) + if (iscntrl(c)) + { c = '?'; + invalid_char = 1; + } + for (char& c : lookup1) if (iscntrl(c)) c = '?'; + for (char& c : lookup2) if (iscntrl(c)) c = '?'; + if (rit1 == rit2) + log << "Unknown region/highway combos " << lookup1 << " and " << lookup2 << " in line: " << trim_line; + else { log << "Unknown region/highway combo "; + log << (rit1 == Route::alt_list_hash.end() ? lookup1 : lookup2); + log << " in line: " << trim_line; + } + if (invalid_char) log << " [contains invalid character(s)]"; + log << '\n'; + splist << orig_line << endlines[l]; + continue; +} +Route* r1 = rit1->second; +Route* r2 = rit2->second; +if (r1->con_route != r2->con_route) +{ log << lookup1 << " and " << lookup2 << " not in same connected route in line: " << trim_line << '\n'; + splist << orig_line << endlines[l]; + continue; +} +if (r1->system->devel()) +{ log << "Ignoring line matching highway in system in development: " << trim_line << '\n'; + splist << orig_line << endlines[l]; + continue; +} +// r1 and r2 are route matches, and we need to find +// waypoint indices, ignoring case and leading +// '+' or '*' when matching +while (*fields[2] == '*' || *fields[2] == '+') fields[2]++; +while (*fields[5] == '*' || *fields[5] == '+') fields[5]++; +upper(fields[2]); +upper(fields[5]); +// look for point indices for labels, first in pri_label_hash +std::unordered_map::iterator lit1 = r1->pri_label_hash.find(fields[2]); +std::unordered_map::iterator lit2 = r2->pri_label_hash.find(fields[5]); +// and then if not found, in alt_label_hash +if (lit1 == r1->pri_label_hash.end()) lit1 = r1->alt_label_hash.find(fields[2]); +if (lit2 == r2->pri_label_hash.end()) lit2 = r2->alt_label_hash.find(fields[5]); +// if we did not find matches for both labels... +if (lit1 == r1->alt_label_hash.end() || lit2 == r2->alt_label_hash.end()) +{ bool invalid_char = 0; + for (char& c : trim_line) + if (iscntrl(c)) + { c = '?'; + invalid_char = 1; + } + for (char* c = fields[2]; *c; c++) if (iscntrl(*c)) *c = '?'; + for (char* c = fields[5]; *c; c++) if (iscntrl(*c)) *c = '?'; + if (lit1 == r1->alt_label_hash.end() && lit2 == r2->alt_label_hash.end()) + log << "Waypoint labels " << fields[2] << " and " << fields[5] << " not found in line: " << trim_line; + else { log << "Waypoint "; + if (lit1 == r1->alt_label_hash.end()) + log << lookup1 << ' ' << fields[2]; + else log << lookup2 << ' ' << fields[5]; + log << " not found in line: " << trim_line; + } + if (invalid_char) log << " [contains invalid character(s)]"; + log << '\n'; + splist << orig_line << endlines[l]; + continue; +} +// are either of the labels used duplicates? +char duplicate = 0; +if (r1->duplicate_labels.find(fields[2]) != r1->duplicate_labels.end()) +{ log << r1->region->code << ": duplicate label " << fields[2] << " in " << r1->root + << ". Please report this error in the TravelMapping forum" + << ". Unable to parse line: " << trim_line << '\n'; + duplicate = 1; +} +if (r2->duplicate_labels.find(fields[5]) != r2->duplicate_labels.end()) +{ log << r2->region->code << ": duplicate label " << fields[5] << " in " << r2->root + << ". Please report this error in the TravelMapping forum" + << ". Unable to parse line: " << trim_line << '\n'; + duplicate = 1; +} +if (duplicate) +{ splist << orig_line << endlines[l]; + continue; +} +bool reverse = 0; +unsigned int index1 = lit1->second; +unsigned int index2 = lit2->second; +// if both region/route combos point to the same chopped route... +if (r1 == r2) + { // if both labels reference the same waypoint... + if (index1 == index2) + { log << "Equivalent waypoint labels mark zero distance traveled in line: " << trim_line << '\n'; + splist << orig_line << endlines[l]; + continue; + } + if (index1 <= index2) + r1->store_traveled_segments(this, index1, index2); + else r1->store_traveled_segments(this, index2, index1); + } +else { if (r1->rootOrder > r2->rootOrder) + { std::swap(r1, r2); + index1 = lit2->second; + index2 = lit1->second; + reverse = 1; + } + // mark the beginning chopped route from index1 to its end + if (r1->is_reversed) + r1->store_traveled_segments(this, 0, index1); + else r1->store_traveled_segments(this, index1, r1->segment_list.size()); + // mark the ending chopped route from its beginning to index2 + if (r2->is_reversed) + r2->store_traveled_segments(this, index2, r2->segment_list.size()); + else r2->store_traveled_segments(this, 0, index2); + // mark any intermediate chopped routes in their entirety. + for (size_t r = r1->rootOrder+1; r < r2->rootOrder; r++) + r1->con_route->roots[r]->store_traveled_segments(this, 0, r1->con_route->roots[r]->segment_list.size()); + } +// both labels are valid; mark in use & proceed +r1->system->lniu_mtx.lock(); +r1->system->listnamesinuse.insert(lookup1); +r1->system->lniu_mtx.unlock(); +r1->system->uarn_mtx.lock(); +r1->system->unusedaltroutenames.erase(lookup1); +r1->system->uarn_mtx.unlock(); +r1->liu_mtx.lock(); +r1->labels_in_use.insert(fields[2]); +r1->liu_mtx.unlock(); +r2->system->lniu_mtx.lock(); +r2->system->listnamesinuse.insert(lookup2); +r2->system->lniu_mtx.unlock(); +r2->system->uarn_mtx.lock(); +r2->system->unusedaltroutenames.erase(lookup2); +r2->system->uarn_mtx.unlock(); +r2->liu_mtx.lock(); +r2->labels_in_use.insert(fields[5]); +r2->liu_mtx.unlock(); +list_entries++; +// new .list lines for region split-ups +if (args->splitregion == r1->region->code || args->splitregion == r2->region->code) +{ + #include "splitregion.cpp" +} +else splist << orig_line << endlines[l]; diff --git a/siteupdate/cplusplus/classes/TravelerList/splitregion.cpp b/siteupdate/cplusplus/classes/TravelerList/splitregion.cpp index 07ab5859..56f1dc70 100644 --- a/siteupdate/cplusplus/classes/TravelerList/splitregion.cpp +++ b/siteupdate/cplusplus/classes/TravelerList/splitregion.cpp @@ -1,24 +1,66 @@ -// new .list lines for region split-ups -if (args->splitregion == upper(fields[0])) -{ // first, comment out original line - splist << "##### " << orig_line << newline; - HighwaySegment *orig_hs = r->segment_list[point_indices[0]]; +Waypoint *w1, *w2; +// comment out original line and indent new line below +splist << "##### " << orig_line << newline << " "; + +// 1st waypoint +if (args->splitregion != r1->region->code) + // if not in splitregion, just use canonical .list name and waypoint label + w1 = r1->point_list[index1]; +else { HighwaySegment *orig_hs = r1->segment_list[index1 == r1->segment_list.size() ? index1-1 : index1]; HighwaySegment *new_hs = 0; if (!orig_hs->concurrent) std::cout << "ERROR: " << orig_hs->str() << " not concurrent" << std::endl; else { size_t count = 0; // find concurrent segment with same name in different region for (HighwaySegment *cs : *(orig_hs->concurrent)) - if (cs->route->name_no_abbrev() == r->name_no_abbrev() && cs->route->region != r->region) + if (cs->route->name_no_abbrev() == r1->name_no_abbrev() && cs->route->region != r1->region) { count++; new_hs = cs; } if (!new_hs) std::cout << "ERROR: concurrent segment not found for " << orig_hs->str() << std::endl; else { if (count > 1) std::cout << "DEBUG: multiple matches found for " << orig_hs->str() << std::endl; - // get lines from associated connected route. // assumption: each chopped route in old full region corresponds 1:1 to a connected route in new chopped regions - splist << new_hs->route->con_route->list_lines(point_indices[0], point_indices[1] - point_indices[0], newline, 2) << endlines[l]; + // index within old chopped route = index within new connected route. + // convert to index within new chopped route + for (long int r = new_hs->route->rootOrder; r > 0; r--) + index1 -= new_hs->route->con_route->roots[r-1]->segment_list.size(); + w1 = new_hs->route->point_list[index1]; } } -} -else splist << orig_line << endlines[l]; + } + +// 2nd waypoint +if (args->splitregion != r2->region->code) + // if not in splitregion, just use canonical .list name and waypoint label + w2 = r2->point_list[index2]; +else { HighwaySegment *orig_hs = r2->segment_list[index2 ? index2-1 : 0]; + HighwaySegment *new_hs = 0; + if (!orig_hs->concurrent) + std::cout << "ERROR: " << orig_hs->str() << " not concurrent" << std::endl; + else { size_t count = 0; + // find concurrent segment with same name in different region + for (HighwaySegment *cs : *(orig_hs->concurrent)) + if (cs->route->name_no_abbrev() == r2->name_no_abbrev() && cs->route->region != r2->region) + { count++; + new_hs = cs; + } + if (!new_hs) std::cout << "ERROR: concurrent segment not found for " << orig_hs->str() << std::endl; + else { if (count > 1) std::cout << "DEBUG: multiple matches found for " << orig_hs->str() << std::endl; + // assumption: each chopped route in old full region corresponds 1:1 to a connected route in new chopped regions + // index within old chopped route = index within new connected route. + // convert to index within new chopped route + for (long int r = new_hs->route->rootOrder; r > 0; r--) + index2 -= new_hs->route->con_route->roots[r-1]->segment_list.size(); + w2 = new_hs->route->point_list[index2]; + } + } + } + +// write new line to file +if (w1->route == w2->route) // 4 fields possible + if (index1 > index2 || reverse) + splist << w2->route->readable_name() << ' ' << w2->label << ' ' << w1->label << endlines[l]; + else splist << w1->route->readable_name() << ' ' << w1->label << ' ' << w2->label << endlines[l]; +else if (reverse) + splist << w2->route->readable_name() << ' ' << w2->label << ' ' << w1->route->readable_name() << ' ' << w1->label << endlines[l]; + else splist << w1->route->readable_name() << ' ' << w1->label << ' ' << w2->route->readable_name() << ' ' << w2->label << endlines[l]; diff --git a/siteupdate/cplusplus/classes/TravelerList/userlog.cpp b/siteupdate/cplusplus/classes/TravelerList/userlog.cpp index 1ea1417e..aa42d40e 100644 --- a/siteupdate/cplusplus/classes/TravelerList/userlog.cpp +++ b/siteupdate/cplusplus/classes/TravelerList/userlog.cpp @@ -1,4 +1,4 @@ -void userlog +void TravelerList::userlog ( ClinchedDBValues *clin_db_val, const double total_active_only_miles, const double total_active_preview_miles, @@ -83,8 +83,7 @@ void userlog unsigned int num_con_rtes_clinched = 0; log << "System " << h->systemname << " by route (traveled routes only):\n"; for (ConnectedRoute &cr : h->con_route_list) - { double con_total_miles = 0; - double con_clinched_miles = 0; + { double con_clinched_miles = 0; std::string to_write = ""; for (Route *r : cr.roots) { // find traveled mileage on this by this user @@ -99,16 +98,11 @@ void userlog con_clinched_miles += miles; to_write += " " + r->readable_name() + ": " + format_clinched_mi(miles,r->mileage) + "\n"; } - con_total_miles += r->mileage; } if (con_clinched_miles) { system_con_umap[&cr] = con_clinched_miles; char clinched = '0'; - /*yDEBUG - if (traveler_name == "oscar" && cr.system->systemname == "usaus" && cr.route == "US85") - printf("\nOscar on US85:\ncon_clinched_miles = %.17f\n con_total_miles = %.17f\n", - con_clinched_miles, con_total_miles);//*/ - if (con_clinched_miles == con_total_miles) + if (con_clinched_miles == cr.mileage) { num_con_rtes_clinched++; clinched = '1'; } @@ -116,7 +110,7 @@ void userlog if (!strchr(fstr, '.')) strcat(fstr, ".0"); clin_db_val->add_ccr("('" + cr.roots[0]->root + "','" + traveler_name + "','" + std::string(fstr) + "','" + clinched + "')"); - log << cr.readable_name() << ": " << format_clinched_mi(con_clinched_miles, con_total_miles) << '\n'; + log << cr.readable_name() << ": " << format_clinched_mi(con_clinched_miles, cr.mileage) << '\n'; if (cr.roots.size() == 1) log << " (" << cr.roots[0]->readable_name() << " only)\n"; else log << to_write << '\n'; @@ -128,7 +122,6 @@ void userlog num_con_rtes_clinched, (int)h->con_route_list.size(), 100*(double)num_con_rtes_clinched/h->con_route_list.size()); log << "System " << h->systemname << fstr << '\n'; con_routes_traveled[h] = system_con_umap; - //#include "debug/oscars_usaus_ConRtes.cpp" } } diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp index 890724bf..d1c49127 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp @@ -13,17 +13,16 @@ bool waypoint_simplification_sort(Waypoint *w1, Waypoint *w2) else return 0; } -const double Waypoint::pi = 3.141592653589793238; - -Waypoint::Waypoint(char *line, Route *rte, std::mutex *strtok_mtx, DatacheckEntryList *datacheckerrors) +Waypoint::Waypoint(char *line, Route *rte, DatacheckEntryList *datacheckerrors) { /* initialize object from a .wpt file line */ route = rte; // parse WPT line - strtok_mtx->lock(); - for (char *token = strtok(line, " "); token; token = strtok(0, " ")) - alt_labels.emplace_back(token); // get all tokens & put into label deque - strtok_mtx->unlock(); + size_t spn = 0; + for (char* c = line; *c; c += spn) + { for (spn = strcspn(c, " "); c[spn] == ' '; spn++) c[spn] = 0; + alt_labels.emplace_back(c); + } // We know alt_labels will have at least one element, because if the WPT line is // blank or contains only spaces, Route::read_wpt will not call this constructor. @@ -306,19 +305,13 @@ bool Waypoint::label_references_route(Route *r, DatacheckEntryList *datacheckerr /* Datacheck */ -inline void Waypoint::duplicate_label(DatacheckEntryList *datacheckerrors, std::unordered_set &all_route_labels) -{ // duplicate labels - // first, check primary label - std::string lower_label = lower(label); - while (lower_label[0] == '+' || lower_label[0] == '*') lower_label.erase(lower_label.begin()); - if (!all_route_labels.insert(lower_label).second) - datacheckerrors->add(route, lower_label, "", "", "DUPLICATE_LABEL", ""); - // then check alt labels - for (std::string &label : alt_labels) - { std::string lower_label = lower(label); - while (lower_label[0] == '+' || lower_label[0] == '*') lower_label.erase(lower_label.begin()); - if (!all_route_labels.insert(lower_label).second) - datacheckerrors->add(route, lower_label, "", "", "DUPLICATE_LABEL", ""); +inline void Waypoint::distance_update(DatacheckEntryList *datacheckerrors, char *fstr, double &vis_dist, Waypoint *prev_w) +{ // visible distance update, and last segment length check + double last_distance = distance_to(prev_w); + vis_dist += last_distance; + if (last_distance > 20) + { sprintf(fstr, "%.2f", last_distance); + datacheckerrors->add(route, prev_w->label, label, "", "LONG_SEGMENT", fstr); } } @@ -337,6 +330,25 @@ inline void Waypoint::duplicate_coords(DatacheckEntryList *datacheckerrors, std: } } +inline void Waypoint::label_invalid_char(DatacheckEntryList *datacheckerrors) +{ // look for labels with invalid characters + if (label == "*") + datacheckerrors->add(route, label, "", "", "LABEL_INVALID_CHAR", ""); + else for (const char *c = label.data(); *c; c++) + if ((*c == 42 || *c == 43) && c > label.data() + || (*c < 40) || (*c == 44) || (*c > 57 && *c < 65) + || (*c == 96) || (*c > 122) || (*c > 90 && *c < 95)) + datacheckerrors->add(route, label, "", "", "LABEL_INVALID_CHAR", ""); + for (std::string& lbl : alt_labels) + if (lbl == "*") + datacheckerrors->add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); + else for (const char *c = lbl.data(); *c; c++) + if (*c == '+' && c > lbl.data() || *c == '*' && (c > lbl.data()+1 || lbl[0] != '+') + || (*c < 40) || (*c == 44) || (*c > 57 && *c < 65) + || (*c == 96) || (*c > 122) || (*c > 90 && *c < 95)) + datacheckerrors->add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); +} + inline bool Waypoint::label_too_long(DatacheckEntryList *datacheckerrors) { // label longer than the DB can store if (label.size() > DBFieldLength::label) @@ -371,35 +383,38 @@ inline void Waypoint::out_of_bounds(DatacheckEntryList *datacheckerrors, char *f } } -inline void Waypoint::distance_update(DatacheckEntryList *datacheckerrors, char *fstr, double &vis_dist, Waypoint *prev_w) -{ // visible distance update, and last segment length check - double last_distance = distance_to(prev_w); - vis_dist += last_distance; - if (last_distance > 20) - { sprintf(fstr, "%.2f", last_distance); - datacheckerrors->add(route, prev_w->label, label, "", "LONG_SEGMENT", fstr); - } -} - /* checks for visible points */ -inline void Waypoint::visible_distance(DatacheckEntryList *datacheckerrors, char *fstr, double &vis_dist, Waypoint *&last_visible) -{ // complete visible distance check, omit report for active - // systems to reduce clutter - if (vis_dist > 10 && !route->system->active()) - { sprintf(fstr, "%.2f", vis_dist); - datacheckerrors->add(route, last_visible->label, label, "", "VISIBLE_DISTANCE", fstr); - } - last_visible = this; - vis_dist = 0; -} - inline void Waypoint::bus_with_i(DatacheckEntryList *datacheckerrors) { // look for I-xx with Bus instead of BL or BS - if (label[0] != 'I' || label[1] != '-') return; - const char *c = label.data()+2; + const char *c = label.data(); + if (*c == '*') c++; + if (*c++ != 'I' || *c++ != '-') return; + if (*c < '0' || *c > '9') return; while (*c >= '0' && *c <= '9') c++; - if (!strncmp(c, "Bus", 3)) datacheckerrors->add(route, label, "", "", "BUS_WITH_I", ""); + if ( *c == 'E' || *c == 'W' || *c == 'C' || *c == 'N' || *c == 'S' + || *c == 'e' || *c == 'w' || *c == 'c' || *c == 'n' || *c == 's' ) c++; + if ( (*c == 'B' || *c == 'b') + && (*(c+1) == 'u' || *(c+1) == 'U') + && (*(c+2) == 's' || *(c+2) == 'S') ) + datacheckerrors->add(route, label, "", "", "BUS_WITH_I", ""); +} + +inline void Waypoint::interstate_no_hyphen(DatacheckEntryList *datacheckerrors) +{ const char *c = label[0] == '*' ? label.data()+1 : label.data(); + if (c[0] == 'T' && c[1] == 'o') c += 2; + if (c[0] == 'I' && isdigit(c[1])) + datacheckerrors->add(route, label, "", "", "INTERSTATE_NO_HYPHEN", ""); +} + +inline void Waypoint::label_invalid_ends(DatacheckEntryList *datacheckerrors) +{ // look for labels with invalid first or final characters + const char *c = label.data(); + while (*c == '*') c++; + if (*c == '_' || *c == '/' || *c == '(') + datacheckerrors->add(route, label, "", "", "INVALID_FIRST_CHAR", std::string(1, *c)); + if (label.back() == '_' || label.back() == '/') + datacheckerrors->add(route, label, "", "", "INVALID_FINAL_CHAR", std::string(1, label.back())); } inline void Waypoint::label_looks_hidden(DatacheckEntryList *datacheckerrors) @@ -415,29 +430,6 @@ inline void Waypoint::label_looks_hidden(DatacheckEntryList *datacheckerrors) datacheckerrors->add(route, label, "", "", "LABEL_LOOKS_HIDDEN", ""); } -inline void Waypoint::label_invalid_char(DatacheckEntryList *datacheckerrors, std::string &lbl) -{ // look for labels with invalid characters - for (const char *c = lbl.data(); *c; c++) - { if (*c < 40) { datacheckerrors->add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); return; } - if (*c == 44) { datacheckerrors->add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); return; } - if (*c > 57 && *c < 65) { datacheckerrors->add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); return; } - if (*c > 90 && *c < 95) { datacheckerrors->add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); return; } - if (*c == 96) { datacheckerrors->add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); return; } - if (*c > 122) { datacheckerrors->add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); return; } - } - if (strpbrk(lbl.data()+1, "+*")) datacheckerrors->add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); -} - -inline void Waypoint::label_invalid_ends(DatacheckEntryList *datacheckerrors) -{ // look for labels with invalid first or final characters - const char *c = label.data(); - while (*c == '*') c++; - if (*c == '_' || *c == '/' || *c == '(') - datacheckerrors->add(route, label, "", "", "INVALID_FIRST_CHAR", std::string(1, *c)); - if (label.back() == '_' || label.back() == '/') - datacheckerrors->add(route, label, "", "", "INVALID_FINAL_CHAR", std::string(1, label.back())); -} - inline void Waypoint::label_parens(DatacheckEntryList *datacheckerrors) { // look for parenthesis balance in label int parens = 0; @@ -461,39 +453,9 @@ inline void Waypoint::label_parens(DatacheckEntryList *datacheckerrors) datacheckerrors->add(route, label, "", "", "LABEL_PARENS", ""); } -inline void Waypoint::underscore_datachecks(DatacheckEntryList *datacheckerrors, const char *slash) -{ const char *underscore = strchr(label.data(), '_'); - if (underscore) - { // look for too many underscores in label - if (strchr(underscore+1, '_')) - datacheckerrors->add(route, label, "", "", "LABEL_UNDERSCORES", ""); - // look for too many characters after underscore in label - if (label.data()+label.size() > underscore+4) - if (label.back() > 'Z' || label.back() < 'A' || label.data()+label.size() > underscore+5) - datacheckerrors->add(route, label, "", "", "LONG_UNDERSCORE", ""); - // look for labels with a slash after an underscore - if (slash > underscore) - datacheckerrors->add(route, label, "", "", "NONTERMINAL_UNDERSCORE", ""); - } -} - -inline void Waypoint::label_slashes(DatacheckEntryList *datacheckerrors, const char *slash) -{ // look for too many slashes in label - if (slash && strchr(slash+1, '/')) - datacheckerrors->add(route, label, "", "", "LABEL_SLASHES", ""); -} - inline void Waypoint::label_selfref(DatacheckEntryList *datacheckerrors, const char *slash) { // looking for the route within the label - //match_start = w.label.find(r.route) - //if match_start >= 0: - // we have a potential match, just need to make sure if the route - // name ends with a number that the matched substring isn't followed - // by more numbers (e.g., NY50 is an OK label in NY5) - // if len(r.route) + match_start == len(w.label) or \ - // not w.label[len(r.route) + match_start].isdigit(): // partially complete "references own route" -- too many FP - //or re.fullmatch('.*/'+r.route+'.*',w.label[w.label) : // first check for number match after a slash, if there is one if (slash && route->route.back() >= '0' && route->route.back() <= '9') { int digit_starts = route->route.size()-1; @@ -526,8 +488,62 @@ inline void Waypoint::label_selfref(DatacheckEntryList *datacheckerrors, const c datacheckerrors->add(route, label, "", "", "LABEL_SELFREF", ""); } -// look for USxxxA but not USxxxAlt, B/Bus (others?) -//if re.fullmatch('US[0-9]+A.*', w.label) and not re.fullmatch('US[0-9]+Alt.*', w.label) or \ -// re.fullmatch('US[0-9]+B.*', w.label) and \ -// not (re.fullmatch('US[0-9]+Bus.*', w.label) or re.fullmatch('US[0-9]+Byp.*', w.label)): -// datacheckerrors.append(DatacheckEntry(r,[w.label],'US_BANNER')) +inline void Waypoint::label_slashes(DatacheckEntryList *datacheckerrors, const char *slash) +{ // look for too many slashes in label + if (slash && strchr(slash+1, '/')) + datacheckerrors->add(route, label, "", "", "LABEL_SLASHES", ""); +} + +inline void Waypoint::lacks_generic(DatacheckEntryList *datacheckerrors) +{ // label lacks generic highway type + const char* c = label[0] == '*' ? label.data()+1 : label.data(); + if ( (*c == 'O' || *c == 'o') + && (*(c+1) == 'l' || *(c+1) == 'L') + && (*(c+2) == 'd' || *(c+2) == 'D') + && *(c+3) >= '0' && *(c+3) <= '9') + datacheckerrors->add(route, label, "", "", "LACKS_GENERIC", ""); +} + +inline void Waypoint::underscore_datachecks(DatacheckEntryList *datacheckerrors, const char *slash) +{ const char *underscore = strchr(label.data(), '_'); + if (underscore) + { // look for too many underscores in label + if (strchr(underscore+1, '_')) + datacheckerrors->add(route, label, "", "", "LABEL_UNDERSCORES", ""); + // look for too many characters after underscore in label + if (label.data()+label.size() > underscore+4) + if (label.back() > 'Z' || label.back() < 'A' || label.data()+label.size() > underscore+5) + datacheckerrors->add(route, label, "", "", "LONG_UNDERSCORE", ""); + // look for labels with a slash after an underscore + if (slash > underscore) + datacheckerrors->add(route, label, "", "", "NONTERMINAL_UNDERSCORE", ""); + } +} + +inline void Waypoint::us_letter(DatacheckEntryList *datacheckerrors) +{ // look for USxxxA but not USxxxAlt, B/Bus/Byp + const char* c = label[0] == '*' ? label.data()+1 : label.data(); + if (*c++ != 'U' || *c++ != 'S') return; + if (*c < '0' || *c++ > '9') return; + while (*c >= '0' && *c <= '9') c++; + if (*c < 'A' || *c++ > 'B') return; + if (*c == 0 || *c == '/' || *c == '_' || *c == '(') + datacheckerrors->add(route, label, "", "", "US_LETTER", ""); + // is it followed by a city abbrev? + else if (*c >= 'A' && *c++ <= 'Z' + && *c >= 'a' && *c++ <= 'z' + && *c >= 'a' && *c++ <= 'z' + && *c == 0 || *c == '/' || *c == '_' || *c == '(') + datacheckerrors->add(route, label, "", "", "US_LETTER", ""); +} + +inline void Waypoint::visible_distance(DatacheckEntryList *datacheckerrors, char *fstr, double &vis_dist, Waypoint *&last_visible) +{ // complete visible distance check, omit report for active + // systems to reduce clutter + if (vis_dist > 10 && !route->system->active()) + { sprintf(fstr, "%.2f", vis_dist); + datacheckerrors->add(route, last_visible->label, label, "", "VISIBLE_DISTANCE", fstr); + } + last_visible = this; + vis_dist = 0; +} diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.h b/siteupdate/cplusplus/classes/Waypoint/Waypoint.h index 694c2c90..c1ebf0ad 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.h +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.h @@ -20,9 +20,8 @@ class Waypoint std::vector ap_coloc; std::forward_list near_miss_points; bool is_hidden; - static const double pi; - Waypoint(char *, Route *, std::mutex *, DatacheckEntryList *); + Waypoint(char *, Route *, DatacheckEntryList *); std::string str(); std::string csv_line(unsigned int); @@ -40,19 +39,21 @@ class Waypoint bool label_references_route(Route *, DatacheckEntryList *); // Datacheck - inline void duplicate_label(DatacheckEntryList *, std::unordered_set &); + inline void distance_update(DatacheckEntryList *, char *, double &, Waypoint *); inline void duplicate_coords(DatacheckEntryList *, std::unordered_set &, char *); + inline void label_invalid_char(DatacheckEntryList *); + inline bool label_too_long(DatacheckEntryList *); inline void out_of_bounds(DatacheckEntryList *, char *); - inline void distance_update(DatacheckEntryList *, char *, double &, Waypoint *); // checks for visible points - inline void visible_distance(DatacheckEntryList *, char *, double &, Waypoint *&); inline void bus_with_i(DatacheckEntryList *); - inline void label_looks_hidden(DatacheckEntryList *); - inline void label_invalid_char(DatacheckEntryList *, std::string &); + inline void interstate_no_hyphen(DatacheckEntryList *); inline void label_invalid_ends(DatacheckEntryList *); + inline void label_looks_hidden(DatacheckEntryList *); inline void label_parens(DatacheckEntryList *); - inline void label_slashes(DatacheckEntryList *, const char *); inline void label_selfref(DatacheckEntryList *, const char *); - inline bool label_too_long(DatacheckEntryList *); + inline void label_slashes(DatacheckEntryList *, const char *); + inline void lacks_generic(DatacheckEntryList *); inline void underscore_datachecks(DatacheckEntryList *, const char *); + inline void us_letter(DatacheckEntryList *); + inline void visible_distance(DatacheckEntryList *, char *, double &, Waypoint *&); }; diff --git a/siteupdate/cplusplus/classes/WaypointQuadtree/WaypointQuadtree.cpp b/siteupdate/cplusplus/classes/WaypointQuadtree/WaypointQuadtree.cpp index b6285089..4d3f8bb5 100644 --- a/siteupdate/cplusplus/classes/WaypointQuadtree/WaypointQuadtree.cpp +++ b/siteupdate/cplusplus/classes/WaypointQuadtree/WaypointQuadtree.cpp @@ -1,4 +1,4 @@ -bool WaypointQuadtree::WaypointQuadtree::refined() +inline bool WaypointQuadtree::WaypointQuadtree::refined() { return nw_child; } @@ -259,7 +259,7 @@ void WaypointQuadtree::write_qt_tmg(std::string filename) std::ofstream tmgfile(filename); tmgfile << "TMG 1.0 simple\n"; tmgfile << vertices.size() << ' ' << edges.size() << '\n'; - for (std::string v : vertices) tmgfile << v << '\n'; - for (std::string e : edges) tmgfile << e << '\n'; + for (std::string& v : vertices) tmgfile << v << '\n'; + for (std::string& e : edges) tmgfile << e << '\n'; tmgfile.close(); } diff --git a/siteupdate/cplusplus/classes/WaypointQuadtree/WaypointQuadtree.h b/siteupdate/cplusplus/classes/WaypointQuadtree/WaypointQuadtree.h index 87da2852..7bba1137 100644 --- a/siteupdate/cplusplus/classes/WaypointQuadtree/WaypointQuadtree.h +++ b/siteupdate/cplusplus/classes/WaypointQuadtree/WaypointQuadtree.h @@ -9,7 +9,7 @@ class WaypointQuadtree unsigned int unique_locations; std::recursive_mutex mtx; - bool refined(); + inline bool refined(); WaypointQuadtree(double, double, double, double); void refine(); void insert(Waypoint*, bool); diff --git a/siteupdate/cplusplus/functions/graph_generation.cpp b/siteupdate/cplusplus/functions/graph_generation.cpp index 02b3e0d6..9b47f0f3 100644 --- a/siteupdate/cplusplus/functions/graph_generation.cpp +++ b/siteupdate/cplusplus/functions/graph_generation.cpp @@ -5,7 +5,7 @@ HighwayGraph graph_data(all_waypoints, highway_systems, datacheckerrors, args.nu cout << et.et() << "Writing graph waypoint simplification log." << endl; ofstream wslogfile(args.logfilepath + "/waypointsimplification.log"); -for (string line : graph_data.waypoint_naming_log) +for (string& line : graph_data.waypoint_naming_log) wslogfile << line << '\n'; wslogfile.close(); graph_data.waypoint_naming_log.clear(); @@ -34,7 +34,8 @@ else { list *regions; graph_vector[0].edges = 0; graph_vector[0].vertices = graph_data.vertices.size(); graph_vector[1].edges = 0; graph_vector[1].vertices = 0; graph_vector[2].edges = 0; graph_vector[2].vertices = 0; - for (std::pair wv : graph_data.vertices) + // get master graph vertex & edge counts for terminal output before writing files + for (std::pair& wv : graph_data.vertices) { graph_vector[0].edges += wv.second->incident_s_edges.size(); if (wv.second->visibility >= 1) { graph_vector[2].vertices++; diff --git a/siteupdate/cplusplus/functions/lower.cpp b/siteupdate/cplusplus/functions/lower.cpp index 228d0cde..1947d2c2 100644 --- a/siteupdate/cplusplus/functions/lower.cpp +++ b/siteupdate/cplusplus/functions/lower.cpp @@ -4,8 +4,8 @@ std::string lower(std::string str) return str; } -char *lower(char *str) -{ for (char *c = str; *c != 0; c++) +const char *lower(const char *str) +{ for (char* c = (char*)str; *c != 0; c++) if (*c >= 'A' && *c <= 'Z') *c += 32; return str; } diff --git a/siteupdate/cplusplus/functions/sql_file.cpp b/siteupdate/cplusplus/functions/sql_file.cpp index ac9c05d0..2317eefb 100644 --- a/siteupdate/cplusplus/functions/sql_file.cpp +++ b/siteupdate/cplusplus/functions/sql_file.cpp @@ -273,7 +273,7 @@ void sqlfile1 first = 1; for (HighwaySystem *h : *highway_systems) if (h->active_or_preview()) - for (std::pair rm : h->mileage_by_region) + for (std::pair& rm : h->mileage_by_region) { if (!first) sqlfile << ','; first = 0; char fstr[35]; @@ -293,7 +293,7 @@ void sqlfile1 sqlfile << "INSERT INTO clinchedOverallMileageByRegion VALUES\n"; first = 1; for (TravelerList *t : *traveler_lists) - for (std::pair rm : t->active_preview_mileage_by_region) + for (std::pair& rm : t->active_preview_mileage_by_region) { if (!first) sqlfile << ','; first = 0; double active_miles = 0; diff --git a/siteupdate/cplusplus/functions/upper.cpp b/siteupdate/cplusplus/functions/upper.cpp index 31a5ae9d..883a1d07 100644 --- a/siteupdate/cplusplus/functions/upper.cpp +++ b/siteupdate/cplusplus/functions/upper.cpp @@ -3,3 +3,9 @@ std::string upper(std::string str) if (str[c] >= 'a' && str[c] <= 'z') str[c] -= 32; return str; } + +const char *upper(const char *str) +{ for (char* c = (char*)str; *c != 0; c++) + if (*c >= 'a' && *c <= 'z') *c -= 32; + return str; +} diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index 90f97f55..94b33094 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -25,6 +25,7 @@ class DatacheckEntryList; class HighwayGraph; class HGVertex; class HGEdge; +#define pi 3.141592653589793238 #include #include #include @@ -43,11 +44,14 @@ class HGEdge; #include #include #include +#include "classes/Arguments.cpp" +#include "classes/ClinchedDBValues.cpp" #include "classes/WaypointQuadtree/WaypointQuadtree.h" #include "classes/Route/Route.h" #include "classes/ConnectedRoute/ConnectedRoute.h" #include "classes/Waypoint/Waypoint.h" #include "classes/HighwaySegment/HighwaySegment.h" +#include "classes/TravelerList/TravelerList.h" #include "classes/GraphGeneration/HGEdge.h" #include "classes/GraphGeneration/PlaceRadius.h" #include "enable_threading.cpp" @@ -61,7 +65,6 @@ class HGEdge; #include "functions/split.cpp" #include "functions/valid_num_str.cpp" #include "classes/DBFieldLength.cpp" -#include "classes/Arguments.cpp" #include "classes/ElapsedTime.cpp" #include "classes/ErrorList.cpp" #include "classes/Region.cpp" @@ -70,7 +73,6 @@ class HGEdge; #include "classes/DatacheckEntryList.cpp" #include "classes/Waypoint/Waypoint.cpp" #include "classes/WaypointQuadtree/WaypointQuadtree.cpp" -#include "classes/ClinchedDBValues.cpp" #include "classes/Route/Route.cpp" #include "classes/ConnectedRoute/ConnectedRoute.cpp" #include "classes/TravelerList/TravelerList.cpp" @@ -95,7 +97,7 @@ using namespace std; int main(int argc, char *argv[]) { ifstream file; string line; - mutex list_mtx, log_mtx, strtok_mtx; + mutex list_mtx; time_t timestamp; // start a timer for including elapsed time reports in messages @@ -242,12 +244,11 @@ int main(int argc, char *argv[]) // deleted on termination of program if (!hs->is_valid) delete hs; else { highway_systems.push_back(hs); - cout << hs->systemname << '.' << std::flush; } } cout << endl; // at the end, print the lines ignored - for (string l : ignoring) cout << l << endl; + for (string& l : ignoring) cout << l << endl; ignoring.clear(); } file.close(); @@ -255,65 +256,6 @@ int main(int argc, char *argv[]) DatacheckEntryList *datacheckerrors = new DatacheckEntryList; // deleted on termination of program - // check for duplicate .list names - // and duplicate root entries among Route and ConnectedRoute - // data in all highway systems - { cout << et.et() << "Checking for duplicate list names in routes, roots in routes and connected routes." << endl; - unordered_set roots, list_names, duplicate_list_names; - unordered_set con_roots; - - for (HighwaySystem *h : highway_systems) - { for (Route& r : h->route_list) - { if (roots.find(r.root) == roots.end()) roots.insert(r.root); - else el.add_error("Duplicate root in route lists: " + r.root); - string list_name = r.readable_name(); - // FIXME branch based on result of list_names.insert - if (list_names.find(list_name) == list_names.end()) list_names.insert(list_name); - else duplicate_list_names.insert(list_name); - } - for (ConnectedRoute& cr : h->con_route_list) - for (size_t r = 0; r < cr.roots.size(); r++) - // FIXME branch based on result of list_names.insert - if (con_roots.find(cr.roots[r]) == con_roots.end()) con_roots.insert(cr.roots[r]); - else el.add_error("Duplicate root in con_route lists: " + cr.roots[r]->root);//*/ - } - - // Make sure every route was listed as a part of some connected route - if (roots.size() == con_roots.size()) - cout << "Check passed: same number of routes as connected route roots. " << roots.size() << endl; - else { el.add_error("Check FAILED: " + to_string(roots.size()) + " routes != " + to_string(con_roots.size()) + " connected route roots."); - // remove con_routes entries from roots - for (Route *cr : con_roots) - { unordered_set::iterator r = roots.find(cr->root); - if (r != roots.end()) roots.erase(r); //FIXME erase by value - } - // there will be some leftovers, let's look up their routes to make - // an error report entry (not worried about efficiency as there would - // only be a few in reasonable cases) - unsigned int num_found = 0; - for (HighwaySystem *h : highway_systems) - for (Route& r : h->route_list) - for (const string& lr : roots) - if (lr == r.root) - { el.add_error("route " + lr + " not matched by any connected route root."); - num_found++; - break; - } - cout << "Added " << num_found << " ROUTE_NOT_IN_CONNECTED error entries." << endl; - } - - // report any duplicate list names as errors - if (duplicate_list_names.empty()) cout << "No duplicate list names found." << endl; - else { for (string d : duplicate_list_names) el.add_error("Duplicate list name: " + d); - cout << "Added " << duplicate_list_names.size() << " DUPLICATE_LIST_NAME error entries." << endl; - } - - roots.clear(); - list_names.clear(); - duplicate_list_names.clear(); - con_roots.clear(); - } - // For tracking whether any .wpt files are in the directory tree // that do not have a .csv file entry that causes them to be // read into the data @@ -335,7 +277,7 @@ int main(int argc, char *argv[]) thread **thr = new thread*[args.numthreads]; for (unsigned int t = 0; t < args.numthreads; t++) thr[t] = new thread(ReadWptThread, t, &highway_systems, &hs_it, &list_mtx, args.highwaydatapath+"/hwy_data", - &el, &all_wpt_files, &all_waypoints, &strtok_mtx, datacheckerrors); + &el, &all_wpt_files, &all_waypoints, datacheckerrors); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); for (unsigned int t = 0; t < args.numthreads; t++) @@ -343,9 +285,9 @@ int main(int argc, char *argv[]) #else for (HighwaySystem* h : highway_systems) { std::cout << h->systemname << std::flush; - for (std::list::iterator r = h->route_list.begin(); r != h->route_list.end(); r++) - { r->read_wpt(&all_waypoints, &el, args.highwaydatapath+"/hwy_data", &strtok_mtx, datacheckerrors, &all_wpt_files); - } + bool usa_flag = h->country->first == "USA"; + for (Route& r : h->route_list) + r.read_wpt(&all_waypoints, &el, args.highwaydatapath+"/hwy_data", usa_flag, datacheckerrors, &all_wpt_files); std::cout << "!" << std::endl; } #endif @@ -424,7 +366,7 @@ int main(int argc, char *argv[]) // if requested, rewrite data with near-miss points merged in if (args.nmpmergepath != "" && !args.errorcheck) - { cout << et.et() << "Writing near-miss point merged wpt files." << endl; //FIXME output dots to indicate progress + { cout << et.et() << "Writing near-miss point merged wpt files." << endl; #ifdef threading_enabled // set up for threaded nmp_merged file writes hs_it = highway_systems.begin(); @@ -436,20 +378,87 @@ int main(int argc, char *argv[]) delete thr[t];//*/ #else for (HighwaySystem *h : highway_systems) - for (Route &r : h->route_list) - r.write_nmp_merged(args.nmpmergepath + "/" + r.rg_str); + { std::cout << h->systemname << std::flush; + for (Route &r : h->route_list) + r.write_nmp_merged(args.nmpmergepath + "/" + r.rg_str); + std::cout << '.' << std::flush; + } #endif + cout << endl; } #include "functions/concurrency_detection.cpp" - // Create hash table for faster lookup of routes by list file name - cout << et.et() << "Creating route hash table for list processing:" << endl; - unordered_map route_hash; - for (HighwaySystem *h : highway_systems) - for (list::iterator r = h->route_list.begin(); r != h->route_list.end(); r++) //FIXME use more compact syntax (&) - { route_hash[lower(r->readable_name())] = &*r; - for (string &a : r->alt_route_names) route_hash[lower(r->rg_str + " " + a)] = &*r; + cout << et.et() << "Processing waypoint labels and checking for unconnected chopped routes." << endl; + for (HighwaySystem* h : highway_systems) + for (Route& r : h->route_list) + { // check for unconnected chopped routes + if (!r.con_route) + el.add_error(r.system->systemname + ".csv: root " + r.root + " not matched by any connected route root."); + + // check for mismatched route endpoints within connected routes + #define q r.con_route->roots[r.rootOrder-1] + if ( r.rootOrder > 0 && q->point_list.size() > 1 && r.point_list.size() > 1 && !r.con_beg()->same_coords(q->con_end()) ) + { if ( q->con_beg()->same_coords(r.con_beg()) ) + { //std::cout << "DEBUG: marking only " << q->str() << " reversed" << std::endl; + //if (q->is_reversed) std::cout << "DEBUG: " << q->str() << " already reversed!" << std::endl; + q->is_reversed = 1; + } + else if ( q->con_end()->same_coords(r.con_end()) ) + { //std::cout << "DEBUG: marking only " << r.str() << " reversed" << std::endl; + //if (r.is_reversed) std::cout << "DEBUG: " << r.str() << " already reversed!" << std::endl; + r.is_reversed = 1; + } + else if ( q->con_beg()->same_coords(r.con_end()) ) + { //std::cout << "DEBUG: marking both " << q->str() << " and " << r.str() << " reversed" << std::endl; + //if (q->is_reversed) std::cout << "DEBUG: " << q->str() << " already reversed!" << std::endl; + //if (r.is_reversed) std::cout << "DEBUG: " << r.str() << " already reversed!" << std::endl; + q->is_reversed = 1; + r.is_reversed = 1; + } + else + { datacheckerrors->add(&r, r.con_beg()->label, "", "", + "DISCONNECTED_ROUTE", q->con_end()->root_at_label()); + datacheckerrors->add(q, q->con_end()->label, "", "", + "DISCONNECTED_ROUTE", r.con_beg()->root_at_label()); + } + } + #undef q + + // create label hashes and check for duplicates + #define w r.point_list[index] + for (unsigned int index = 0; index < r.point_list.size(); index++) + { // ignore case and leading '+' or '*' + std::string upper_label = upper(w->label); + while (upper_label[0] == '+' || upper_label[0] == '*') + upper_label = upper_label.substr(1); + // if primary label not duplicated, add to r.pri_label_hash + if (r.alt_label_hash.find(upper_label) != r.alt_label_hash.end()) + { datacheckerrors->add(&r, upper_label, "", "", "DUPLICATE_LABEL", ""); + r.duplicate_labels.insert(upper_label); + } + else if (!r.pri_label_hash.insert(std::pair(upper_label, index)).second) + { datacheckerrors->add(&r, upper_label, "", "", "DUPLICATE_LABEL", ""); + r.duplicate_labels.insert(upper_label); + } + for (std::string& a : w->alt_labels) + { // create canonical AltLabels + while (a[0] == '+' || a[0] == '*') a = a.substr(1); + upper(a.data()); + // populate unused set + r.unused_alt_labels.insert(a); + // create label->index hashes and check if AltLabels duplicated + if (r.pri_label_hash.find(a) != r.pri_label_hash.end()) + { datacheckerrors->add(&r, a, "", "", "DUPLICATE_LABEL", ""); + r.duplicate_labels.insert(a); + } + else if (!r.alt_label_hash.insert(std::pair(a, index)).second) + { datacheckerrors->add(&r, a, "", "", "DUPLICATE_LABEL", ""); + r.duplicate_labels.insert(a); + } + } + } + #undef w } // Create a list of TravelerList objects, one per person @@ -460,7 +469,7 @@ int main(int argc, char *argv[]) #ifdef threading_enabled // set up for threaded .list file processing for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(ReadListThread, t, &traveler_ids, &id_it, &traveler_lists, &list_mtx, &strtok_mtx, &args, &route_hash, &el); + thr[t] = new thread(ReadListThread, t, &traveler_ids, &id_it, &traveler_lists, &list_mtx, &args, &el); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); for (unsigned int t = 0; t < args.numthreads; t++) @@ -468,7 +477,7 @@ int main(int argc, char *argv[]) #else for (string &t : traveler_ids) { cout << t << ' ' << std::flush; - traveler_lists.push_back(new TravelerList(t, &route_hash, &el, &args, &strtok_mtx)); + traveler_lists.push_back(new TravelerList(t, &el, &args)); } traveler_ids.clear(); #endif @@ -481,8 +490,16 @@ int main(int argc, char *argv[]) travnum++; } - //#include "debug/highway_segment_log.cpp" - //#include "debug/pioneers.cpp" + cout << et.et() << "Clearing route & label hash tables." << endl; + Route::root_hash.clear(); + Route::pri_list_hash.clear(); + Route::alt_list_hash.clear(); + for (HighwaySystem* h : highway_systems) + for (Route& r : h->route_list) + { r.pri_label_hash.clear(); + r.alt_label_hash.clear(); + r.duplicate_labels.clear(); + } // Read updates.csv file, just keep in the fields list for now since we're // just going to drop this into the DB later anyway @@ -571,47 +588,72 @@ int main(int argc, char *argv[]) } file.close(); - // write log file for points in use -- might be more useful in the DB later, - // or maybe in another format - cout << et.et() << "Writing points in use log." << endl; - ofstream inusefile(args.logfilepath+"/pointsinuse.log"); - timestamp = time(0); - inusefile << "Log file created at: " << ctime(×tamp); - for (HighwaySystem *h : highway_systems) - for (Route &r : h->route_list) - if (r.labels_in_use.size()) - { inusefile << r.root << '(' << r.point_list.size() << "):"; - list liu_list(r.labels_in_use.begin(), r.labels_in_use.end()); - liu_list.sort(); - for (string label : liu_list) inusefile << ' ' << label; - inusefile << '\n'; - r.labels_in_use.clear(); - } - inusefile.close(); - - // write log file for alt labels not in use - cout << et.et() << "Writing unused alt labels log." << endl; + cout << et.et() << "Writing pointsinuse.log, unusedaltlabels.log, listnamesinuse.log and unusedaltroutenames.log" << endl; unsigned int total_unused_alt_labels = 0; + unsigned int total_unusedaltroutenames = 0; list unused_alt_labels; + ofstream piufile(args.logfilepath+"/pointsinuse.log"); + ofstream lniufile(args.logfilepath+"/listnamesinuse.log"); + ofstream uarnfile(args.logfilepath+"/unusedaltroutenames.log"); + timestamp = time(0); + piufile << "Log file created at: " << ctime(×tamp); + lniufile << "Log file created at: " << ctime(×tamp); + uarnfile << "Log file created at: " << ctime(×tamp); for (HighwaySystem *h : highway_systems) - for (Route &r : h->route_list) + { for (Route &r : h->route_list) + { // labelsinuse.log line + if (r.labels_in_use.size()) + { piufile << r.root << '(' << r.point_list.size() << "):"; + list liu_list(r.labels_in_use.begin(), r.labels_in_use.end()); + liu_list.sort(); + for (string& label : liu_list) piufile << ' ' << label; + piufile << '\n'; + r.labels_in_use.clear(); + } + // unusedaltlabels.log lines, to be sorted by root later if (r.unused_alt_labels.size()) { total_unused_alt_labels += r.unused_alt_labels.size(); string ual_entry = r.root + '(' + to_string(r.unused_alt_labels.size()) + "):"; list ual_list(r.unused_alt_labels.begin(), r.unused_alt_labels.end()); ual_list.sort(); - for (string label : ual_list) ual_entry += ' ' + label; - r.unused_alt_labels.clear(); + for (string& label : ual_list) ual_entry += ' ' + label; unused_alt_labels.push_back(ual_entry); + r.unused_alt_labels.clear(); } + } + // listnamesinuse.log line + if (h->listnamesinuse.size()) + { lniufile << h->systemname << '(' << h->route_list.size() << "):"; + list lniu_list(h->listnamesinuse.begin(), h->listnamesinuse.end()); + lniu_list.sort(); + for (string& list_name : lniu_list) lniufile << " \"" << list_name << '"'; + lniufile << '\n'; + h->listnamesinuse.clear(); + } + // unusedaltroutenames.log line + if (h->unusedaltroutenames.size()) + { total_unusedaltroutenames += h->unusedaltroutenames.size(); + uarnfile << h->systemname << '(' << h->unusedaltroutenames.size() << "):"; + list uarn_list(h->unusedaltroutenames.begin(), h->unusedaltroutenames.end()); + uarn_list.sort(); + for (string& list_name : uarn_list) uarnfile << " \"" << list_name << '"'; + uarnfile << '\n'; + h->unusedaltroutenames.clear(); + } + } + piufile.close(); + lniufile.close(); + uarnfile << "Total: " << total_unusedaltroutenames << '\n'; + uarnfile.close(); + // sort lines and write unusedaltlabels.log unused_alt_labels.sort(); - ofstream unusedfile(args.logfilepath+"/unusedaltlabels.log"); + ofstream ualfile(args.logfilepath+"/unusedaltlabels.log"); timestamp = time(0); - unusedfile << "Log file created at: " << ctime(×tamp); - for (string &ual_entry : unused_alt_labels) unusedfile << ual_entry << '\n'; + ualfile << "Log file created at: " << ctime(×tamp); + for (string &ual_entry : unused_alt_labels) ualfile << ual_entry << '\n'; unused_alt_labels.clear(); - unusedfile << "Total: " << total_unused_alt_labels << '\n'; - unusedfile.close(); + ualfile << "Total: " << total_unused_alt_labels << '\n'; + ualfile.close(); // now augment any traveler clinched segments for concurrencies @@ -623,7 +665,7 @@ int main(int argc, char *argv[]) list::iterator tl_it = traveler_lists.begin(); for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(ConcAugThread, t, &traveler_lists, &tl_it, &list_mtx, &log_mtx, augment_lists+t); + thr[t] = new thread(ConcAugThread, t, &traveler_lists, &tl_it, &list_mtx, augment_lists+t); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); cout << "!\n" << et.et() << "Writing to concurrencies.log." << endl; @@ -710,7 +752,7 @@ int main(int argc, char *argv[]) region_entries.push_back(region->code + fstr); } region_entries.sort(); - for (string e : region_entries) hdstatsfile << e; + for (string& e : region_entries) hdstatsfile << e; for (HighwaySystem *h : highway_systems) { sprintf(fstr, ") total: %.2f mi\n", h->total_mileage()); @@ -718,7 +760,7 @@ int main(int argc, char *argv[]) if (h->mileage_by_region.size() > 1) { hdstatsfile << "System " << h->systemname << " by region:\n"; list regions_in_system; - for (pair rm : h->mileage_by_region) + for (pair& rm : h->mileage_by_region) regions_in_system.push_back(rm.first); regions_in_system.sort(sort_regions_by_code); for (Region *r : regions_in_system) @@ -727,19 +769,17 @@ int main(int argc, char *argv[]) } } hdstatsfile << "System " << h->systemname << " by route:\n"; - for (list::iterator cr = h->con_route_list.begin(); cr != h->con_route_list.end(); cr++) - { double con_total_miles = 0; - string to_write = ""; - for (Route *r : cr->roots) + for (ConnectedRoute& cr : h->con_route_list) + { string to_write = ""; + for (Route *r : cr.roots) { sprintf(fstr, ": %.2f mi\n", r->mileage); to_write += " " + r->readable_name() + fstr; - con_total_miles += r->mileage; + cr.mileage += r->mileage; } - cr->mileage = con_total_miles; //FIXME? - sprintf(fstr, ": %.2f mi", con_total_miles); - hdstatsfile << cr->readable_name() << fstr; - if (cr->roots.size() == 1) - hdstatsfile << " (" << cr->roots[0]->readable_name() << " only)\n"; + sprintf(fstr, ": %.2f mi", cr.mileage); + hdstatsfile << cr.readable_name() << fstr; + if (cr.roots.size() == 1) + hdstatsfile << " (" << cr.roots[0]->readable_name() << " only)\n"; else hdstatsfile << '\n' << to_write; } } @@ -787,7 +827,7 @@ int main(int argc, char *argv[]) allfile << '\n'; for (TravelerList *t : traveler_lists) { double t_total_mi = 0; - for (std::pair rm : t->active_only_mileage_by_region) + for (std::pair& rm : t->active_only_mileage_by_region) t_total_mi += rm.second; sprintf(fstr, "%.2f", t_total_mi); allfile << t->traveler_name << ',' << fstr; @@ -825,7 +865,7 @@ int main(int argc, char *argv[]) allfile << '\n'; for (TravelerList *t : traveler_lists) { double t_total_mi = 0; - for (std::pair rm : t->active_preview_mileage_by_region) + for (std::pair& rm : t->active_preview_mileage_by_region) t_total_mi += rm.second; sprintf(fstr, "%.2f", t_total_mi); allfile << t->traveler_name << ',' << fstr; @@ -853,7 +893,7 @@ int main(int argc, char *argv[]) sysfile << "Traveler,Total"; regions.clear(); total_mi = 0; - for (std::pair rm : h->mileage_by_region) + for (std::pair& rm : h->mileage_by_region) { regions.push_back(rm.first); total_mi += rm.second; //TODO is this right? } @@ -889,17 +929,23 @@ int main(int argc, char *argv[]) cout << et.et() << "Reading datacheckfps.csv." << endl; file.open(args.highwaydatapath+"/datacheckfps.csv"); getline(file, line); // ignore header line - list> datacheckfps; //FIXME try implementing as an unordered_multiset; see if speed increases + list> datacheckfps; unordered_set datacheck_always_error - ({ "BAD_ANGLE", "DUPLICATE_LABEL", "HIDDEN_TERMINUS", + ({ "BAD_ANGLE", "DISCONNECTED_ROUTE", "DUPLICATE_LABEL", + "HIDDEN_TERMINUS", "INTERSTATE_NO_HYPHEN", "INVALID_FINAL_CHAR", "INVALID_FIRST_CHAR", "LABEL_INVALID_CHAR", "LABEL_PARENS", "LABEL_SLASHES", "LABEL_TOO_LONG", "LABEL_UNDERSCORES", "LONG_UNDERSCORE", "MALFORMED_LAT", "MALFORMED_LON", "MALFORMED_URL", - "NONTERMINAL_UNDERSCORE" + "NONTERMINAL_UNDERSCORE", "US_LETTER" }); while (getline(file, line)) - { if (line.back() == 0x0D) line.erase(line.end()-1); // trim DOS newlines + { // trim DOS newlines & trailing whitespace + while (line.back() == 0x0D || line.back() == ' ' || line.back() == '\t') + line.pop_back(); + // trim leading whitespace + while (line[0] == ' ' || line[0] == '\t') + line = line.substr(1); if (line.empty()) continue; // parse system updates.csv line size_t NumFields = 6; @@ -935,28 +981,27 @@ int main(int argc, char *argv[]) fpfile << "Log file created at: " << ctime(×tamp); unsigned int counter = 0; unsigned int fpcount = 0; - for (list::iterator d = datacheckerrors->entries.begin(); d != datacheckerrors->entries.end(); d++) + for (DatacheckEntry& d : datacheckerrors->entries) { //cout << "Checking: " << d->str() << endl; counter++; if (counter % 1000 == 0) cout << '.' << flush; for (list>::iterator fp = datacheckfps.begin(); fp != datacheckfps.end(); fp++) - if (d->match_except_info(*fp)) - if (d->info == (*fp)[5]) + if (d.match_except_info(*fp)) + if (d.info == (*fp)[5]) { //cout << "Match!" << endl; - d->fp = 1; + d.fp = 1; fpcount++; datacheckfps.erase(fp); break; } else { fpfile << "FP_ENTRY: " << (*fp)[0] << ';' << (*fp)[1] << ';' << (*fp)[2] << ';' << (*fp)[3] << ';' << (*fp)[4] << ';' << (*fp)[5] << '\n'; - fpfile << "CHANGETO: " << (*fp)[0] << ';' << (*fp)[1] << ';' << (*fp)[2] << ';' << (*fp)[3] << ';' << (*fp)[4] << ';' << d->info << '\n'; + fpfile << "CHANGETO: " << (*fp)[0] << ';' << (*fp)[1] << ';' << (*fp)[2] << ';' << (*fp)[3] << ';' << (*fp)[4] << ';' << d.info << '\n'; } } fpfile.close(); cout << '!' << endl; - cout << et.et() << "Found " << datacheckerrors->entries.size() << " datacheck errors." << endl; - cout << et.et() << "Matched " << fpcount << " FP entries." << endl; + cout << et.et() << "Found " << datacheckerrors->entries.size() << " datacheck errors and matched " << fpcount << " FP entries." << endl; // write log of unmatched false positives from the datacheckfps.csv cout << et.et() << "Writing log of unmatched datacheck FP entries." << endl; @@ -964,8 +1009,8 @@ int main(int argc, char *argv[]) timestamp = time(0); fpfile << "Log file created at: " << ctime(×tamp); if (datacheckfps.empty()) fpfile << "No unmatched FP entries.\n"; - else for (array entry : datacheckfps) - fpfile << entry[0] << ';' << entry[1] << ';' << entry[2] << ';' << entry[3] << ';' << entry[4] << ';' << entry[5] << '\n'; + else for (array& entry : datacheckfps) + fpfile << entry[0] << ';' << entry[1] << ';' << entry[2] << ';' << entry[3] << ';' << entry[4] << ';' << entry[5] << '\n'; fpfile.close(); // datacheck.log file @@ -1053,7 +1098,7 @@ int main(int argc, char *argv[]) unsigned other_count = 0; unsigned int total_rtes = 0; for (HighwaySystem *h : highway_systems) - for (Route r : h->route_list) + for (Route& r : h->route_list) { total_rtes++; if (h->devel()) d_count++; else { if (h->active()) a_count++; diff --git a/siteupdate/cplusplus/threads/ConcAugThread.cpp b/siteupdate/cplusplus/threads/ConcAugThread.cpp index 9a848185..f7c3b5ef 100644 --- a/siteupdate/cplusplus/threads/ConcAugThread.cpp +++ b/siteupdate/cplusplus/threads/ConcAugThread.cpp @@ -1,5 +1,5 @@ void ConcAugThread(unsigned int id, std::list *travlists, std::list::iterator *it, - std::mutex *tl_mtx, std::mutex *log_mtx, std::list* augment_list) + std::mutex *tl_mtx, std::list* augment_list) { //printf("Starting ConcAugThread %02i\n", id); fflush(stdout); while (*it != travlists->end()) { tl_mtx->lock(); diff --git a/siteupdate/cplusplus/threads/NmpMergedThread.cpp b/siteupdate/cplusplus/threads/NmpMergedThread.cpp index 2599e7a6..1ddeaf67 100644 --- a/siteupdate/cplusplus/threads/NmpMergedThread.cpp +++ b/siteupdate/cplusplus/threads/NmpMergedThread.cpp @@ -11,6 +11,7 @@ void NmpMergedThread(unsigned int id, std::list *hs_list, std::l (*it)++; //printf("NmpMergedThread %02i (*it)++\n", id); fflush(stdout); mtx->unlock(); + std::cout << h->systemname << '.' << std::flush; for (Route &r : h->route_list) r.write_nmp_merged(*nmpmergepath + "/" + r.rg_str); } diff --git a/siteupdate/cplusplus/threads/ReadListThread.cpp b/siteupdate/cplusplus/threads/ReadListThread.cpp index 5adaa85c..f48f1af9 100644 --- a/siteupdate/cplusplus/threads/ReadListThread.cpp +++ b/siteupdate/cplusplus/threads/ReadListThread.cpp @@ -1,7 +1,6 @@ void ReadListThread ( unsigned int id, std::list *traveler_ids, std::list::iterator *it, - std::list *traveler_lists, std::mutex *tl_mtx, std::mutex *strtok_mtx, - Arguments *args, std::unordered_map *route_hash, ErrorList *el + std::list *traveler_lists, std::mutex *tl_mtx, Arguments *args, ErrorList *el ) { //printf("Starting ReadListThread %02i\n", id); fflush(stdout); while (*it != traveler_ids->end()) @@ -16,7 +15,7 @@ void ReadListThread //printf("ReadListThread %02i (*it)++\n", id); fflush(stdout); std::cout << tl << ' ' << std::flush; tl_mtx->unlock(); - TravelerList *t = new TravelerList(tl, route_hash, el, args, strtok_mtx); + TravelerList *t = new TravelerList(tl, el, args); // deleted on termination of program TravelerList::alltrav_mtx.lock(); traveler_lists->push_back(t); diff --git a/siteupdate/cplusplus/threads/ReadWptThread.cpp b/siteupdate/cplusplus/threads/ReadWptThread.cpp index e4c00de8..f8a20099 100644 --- a/siteupdate/cplusplus/threads/ReadWptThread.cpp +++ b/siteupdate/cplusplus/threads/ReadWptThread.cpp @@ -1,7 +1,7 @@ void ReadWptThread ( unsigned int id, std::list *hs_list, std::list::iterator *it, std::mutex *hs_mtx, std::string path, ErrorList *el, std::unordered_set *all_wpt_files, - WaypointQuadtree *all_waypoints, std::mutex *strtok_mtx, DatacheckEntryList *datacheckerrors + WaypointQuadtree *all_waypoints, DatacheckEntryList *datacheckerrors ) { //printf("Starting ReadWptThread %02i\n", id); fflush(stdout); while (*it != hs_list->end()) @@ -16,8 +16,9 @@ void ReadWptThread //printf("ReadWptThread %02i (*it)++\n", id); fflush(stdout); hs_mtx->unlock(); std::cout << h->systemname << std::flush; + bool usa_flag = h->country->first == "USA"; for (Route &r : h->route_list) - r.read_wpt(all_waypoints, el, path, strtok_mtx, datacheckerrors, all_wpt_files); + r.read_wpt(all_waypoints, el, path, usa_flag, datacheckerrors, all_wpt_files); std::cout << "!" << std::endl; } } diff --git a/siteupdate/python-teresco/localupdate.sh b/siteupdate/python-teresco/localupdate.sh index 8e7af241..5e58c56f 100644 --- a/siteupdate/python-teresco/localupdate.sh +++ b/siteupdate/python-teresco/localupdate.sh @@ -121,7 +121,7 @@ mv TravelMapping-$datestr.sql $tmpdir echo "$0: sending email notification" mailx -s "Travel Mapping Site Update Complete" travelmapping-siteupdates@teresco.org < " + str(DBFieldLength.city) + " bytes in " + system.systemname + ".csv line: " + line) - self.root = fields[6] + self.root = fields[6].lower() if len(self.root.encode('utf-8')) > DBFieldLength.root: el.add_error("Root > " + str(DBFieldLength.root) + " bytes in " + system.systemname + ".csv line: " + line) - self.alt_route_names = fields[7].split(",") + if len(fields[7]) == 0: + self.alt_route_names = [] + else: + self.alt_route_names = fields[7].upper().split(",") + + # insert into root_hash, checking for duplicate root entries + if self.root in Route.root_hash: + el.add_error("Duplicate root in " + system.systemname + ".csv: " + self.root + + " already in " + Route.root_hash[self.root].system.systemname + ".csv") + else: + Route.root_hash[self.root] = self + # insert list name into pri_list_hash, checking for duplicate .list names + list_name = self.readable_name().upper() + if list_name in Route.alt_list_hash: + el.add_error("Duplicate main list name in " + self.root + ": '" + self.readable_name() + + "' already points to " + Route.alt_list_hash[list_name].root) + elif list_name in Route.pri_list_hash: + el.add_error("Duplicate main list name in " + self.root + ": '" + self.readable_name() + + "' already points to " + Route.pri_list_hash[list_name].root) + else: + Route.pri_list_hash[list_name] = self + # insert alt names into alt_list_hash, checking for duplicate .list names + for a in self.alt_route_names: + list_name = self.region.upper() + ' ' + a + if list_name in Route.pri_list_hash: + el.add_error("Duplicate alt route name in " + self.root + ": '" + self.region + ' ' + a + + "' already points to " + Route.pri_list_hash[list_name].root) + elif list_name in Route.alt_list_hash: + el.add_error("Duplicate alt route name in " + self.root + ": '" + self.region + ' ' + a + + "' already points to " + Route.alt_list_hash[list_name].root) + else: + Route.alt_list_hash[list_name] = self + # populate unused set + system.unusedaltroutenames.add(list_name) + self.point_list = [] self.labels_in_use = set() self.unused_alt_labels = set() + self.duplicate_labels = set() + self.pri_label_hash = dict() + self.alt_label_hash = dict() self.segment_list = [] + self.con_route = None self.mileage = 0.0 self.rootOrder = -1 # order within connected route + self.is_reversed = False def __str__(self): """printable version of the object""" @@ -976,9 +1015,6 @@ def read_wpt(self,all_waypoints,all_waypoints_lock,datacheckerrors,el,path="../. w = previous_point continue self.point_list.append(w) - # populate unused alt labels - for label in w.alt_labels: - self.unused_alt_labels.add(label.upper().strip("+")) all_waypoints_lock.acquire() # look for near-miss points (before we add this one in) @@ -1009,6 +1045,7 @@ def read_wpt(self,all_waypoints,all_waypoints_lock,datacheckerrors,el,path="../. self.segment_list.append(HighwaySegment(previous_point, w, self)) if len(self.point_list) < 2: el.add_error("Route contains fewer than 2 points: " + str(self)) + print(".", end="",flush=True) def print_route(self): for point in self.point_list: @@ -1049,6 +1086,19 @@ def clinched_by_traveler(self,t): miles += s.length return miles + def store_traveled_segments(self, t, beg, end): + # store clinched segments with traveler and traveler with segments + for pos in range(beg, end): + hs = self.segment_list[pos] + hs.add_clinched_by(t) + t.clinched_segments.add(hs) + + def con_beg(self): + return self.point_list[-1] if self.is_reversed else self.point_list[0] + + def con_end(self): + return self.point_list[0] if self.is_reversed else self.point_list[-1] + class ConnectedRoute: """This class encapsulates a single 'connected route' as given by a single line of a _con.csv file @@ -1086,22 +1136,24 @@ def __init__(self,line,system,el): "_con.csv line: " + line) # fields[4] is the list of roots, which will become a python list # of Route objects already in the system - roots = fields[4].split(",") rootOrder = 0 - for root in roots: - route = None - for check_route in system.route_list: - if check_route.root == root: - route = check_route - break - if route is None: - el.add_error("Could not find Route matching ConnectedRoute root " + root + - " in system " + system.systemname + '.') - else: + for root in fields[4].lower().split(","): + try: + route = Route.root_hash[root] self.roots.append(route) + if route.con_route is not None: + el.add_error("Duplicate root in " + system.systemname + "_con.csv: " + route.root + + " already in " + route.con_route.system.systemname + "_con.csv") + if system != route.system: + el.add_error("System mismatch: chopped route " + route.root + " from " + route.system.systemname + + ".csv in connected route in " + system.systemname + "_con.csv"); + route.con_route = self # save order of route in connected route route.rootOrder = rootOrder - rootOrder += 1 + rootOrder += 1 + except KeyError: + el.add_error("Could not find Route matching ConnectedRoute root " + root + + " in system " + system.systemname + '.') if len(self.roots) < 1: el.add_error("No roots in " + system.systemname + "_con.csv line: " + line) # will be computed for routes in active & preview systems later @@ -1146,6 +1198,8 @@ def __init__(self,systemname,country,fullname,color,tier,level,el, self.level = level self.mileage_by_region = dict() self.vertices = set() + self.listnamesinuse = set() + self.unusedaltroutenames = set() # read chopped routes .csv try: @@ -1214,7 +1268,7 @@ class TravelerList: start_waypoint end_waypoint """ - def __init__(self,travelername,route_hash,el,path="../../../UserData/list_files"): + def __init__(self,travelername,el,path="../../../UserData/list_files"): list_entries = 0 self.clinched_segments = set() self.traveler_name = travelername[:-5] @@ -1233,72 +1287,240 @@ def __init__(self,travelername,route_hash,el,path="../../../UserData/list_files" if len(line) == 0 or line.startswith("#"): continue fields = re.split('[ \t]+',line) - if len(fields) != 4: - # OK if 5th field exists and starts with # - if len(fields) < 5 or not fields[4].startswith("#"): - self.log_entries.append("Incorrect format line: " + line) - continue - - # find the root that matches in some system and when we do, match labels - route_entry = fields[1].lower() - lookup = fields[0].lower() + ' ' + route_entry - if lookup not in route_hash: - (line, invchar) = no_control_chars(line) - self.log_entries.append("Unknown region/highway combo in line: " + line) - if invchar: - self.log_entries[-1] += " [contains invalid character(s)]" - else: - r = route_hash[lookup] - for a in r.alt_route_names: - if route_entry == a.lower(): - self.log_entries.append("Note: deprecated route name " + fields[1] + " -> canonical name " + r.list_entry_name() + " in line " + line) - break - + # truncate inline comments from fields list + for i in range(min(len(fields), 7)): + if fields[i][0] == '#': + fields = fields[:i] + break + if len(fields) == 4: + # find the route that matches and when we do, match labels + lookup = fields[0].upper() + ' ' + fields[1].upper() + # look for region/route combo, first in pri_list_hash + try: + r = Route.pri_list_hash[lookup] + # and then if not found, in alt_list_hash + except KeyError: + try: + r = Route.alt_list_hash[lookup] + except KeyError: + (line, invchar) = no_control_chars(line) + self.log_entries.append("Unknown region/highway combo in line: " + line) + if invchar: + self.log_entries[-1] += " [contains invalid character(s)]" + continue + else: + self.log_entries.append("Note: deprecated route name " + fields[1] + " -> canonical name " + r.list_entry_name() + " in line: " + line) if r.system.devel(): self.log_entries.append("Ignoring line matching highway in system in development: " + line) continue - # r is a route match, r.root is our root, and we need to find - # canonical waypoint labels, ignoring case and leading + # r is a route match, and we need to find + # waypoint indices, ignoring case and leading # "+" or "*" when matching - point_indices = [] - checking_index = 0; - for w in r.point_list: - lower_label = w.label.lower().lstrip("+*") - list_label_1 = fields[2].lower().lstrip("+*") - list_label_2 = fields[3].lower().lstrip("+*") - if list_label_1 == lower_label or list_label_2 == lower_label: - point_indices.append(checking_index) - r.labels_in_use.add(lower_label.upper()) - else: - for alt in w.alt_labels: - lower_label = alt.lower().strip("+") - if list_label_1 == lower_label or list_label_2 == lower_label: - point_indices.append(checking_index) - r.labels_in_use.add(lower_label.upper()) - # if we have not yet used this alt label, remove it from the unused list - if lower_label.upper() in r.unused_alt_labels: - r.unused_alt_labels.remove(lower_label.upper()) - - checking_index += 1 - if len(point_indices) != 2: + list_label_1 = fields[2].lstrip("+*").upper() + list_label_2 = fields[3].lstrip("+*").upper() + + # look for point indices for labels, first in pri_label_hash + # and then if not found, in alt_label_hash + try: + index1 = r.pri_label_hash[list_label_1] + except KeyError: + try: + index1 = r.alt_label_hash[list_label_1] + except KeyError: + index1 = None + try: + index2 = r.pri_label_hash[list_label_2] + except KeyError: + try: + index2 = r.alt_label_hash[list_label_2] + except KeyError: + index2 = None + # if we did not find matches for both labels... + if index1 is None or index2 is None: + (list_label_1, invchar) = no_control_chars(list_label_1) + (list_label_2, invchar) = no_control_chars(list_label_2) (line, invchar) = no_control_chars(line) - self.log_entries.append("Waypoint label(s) not found in line: " + line) + if index1 == index2: + self.log_entries.append("Waypoint labels " + list_label_1 + " and " \ + + list_label_2 + " not found in line: " + line) + else: + log_entry = "Waypoint label " + log_entry += list_label_1 if index1 is None else list_label_2 + log_entry += " not found in line: " + line + self.log_entries.append(log_entry) if invchar: self.log_entries[-1] += " [contains invalid character(s)]" + continue + # are either of the labels used duplicates? + duplicate = False + if list_label_1 in r.duplicate_labels: + log_entry = r.region + ": duplicate label " + list_label_1 + " in " + r.root \ + + ". Please report this error in the TravelMapping forum" \ + + ". Unable to parse line: " + line + self.log_entries.append(log_entry) + duplicate = True + if list_label_2 in r.duplicate_labels: + log_entry = r.region + ": duplicate label " + list_label_2 + " in " + r.root \ + + ". Please report this error in the TravelMapping forum" \ + + ". Unable to parse line: " + line + self.log_entries.append(log_entry) + duplicate = True + if duplicate: + continue + # if both labels reference the same waypoint... + if index1 == index2: + self.log_entries.append("Equivalent waypoint labels mark zero distance traveled in line: " + line) + # otherwise both labels are valid; mark in use & proceed else: + r.system.listnamesinuse.add(lookup) + r.system.unusedaltroutenames.discard(lookup) + r.labels_in_use.add(list_label_1) + r.labels_in_use.add(list_label_2) + r.unused_alt_labels.discard(list_label_1) + r.unused_alt_labels.discard(list_label_2) list_entries += 1 - # find the segments we just matched and store this traveler with the - # segments and the segments with the traveler (might not need both - # ultimately) - for wp_pos in range(point_indices[0],point_indices[1]): - hs = r.segment_list[wp_pos] - hs.add_clinched_by(self) - if hs not in self.clinched_segments: - self.clinched_segments.add(hs) + if index1 > index2: + (index1, index2) = (index2, index1) + r.store_traveled_segments(self, index1, index2) + elif len(fields) == 6: + lookup1 = fields[0].upper() + ' ' + fields[1].upper() + lookup2 = fields[3].upper() + ' ' + fields[4].upper() + # look for region/route combos, first in pri_list_hash + # and then if not found, in alt_list_hash + both_lookups_found = True + try: + r1 = Route.pri_list_hash[lookup1] + except KeyError: + try: + r1 = Route.alt_list_hash[lookup1] + except KeyError: + r1 = None + else: + self.log_entries.append("Note: deprecated route name \"" + fields[0] + ' ' + fields[1] \ + + "\" -> canonical name \"" + r1.readable_name() + "\" in line: " + line) + try: + r2 = Route.pri_list_hash[lookup2] + except KeyError: + try: + r2 = Route.alt_list_hash[lookup2] + except KeyError: + r2 = None + else: + self.log_entries.append("Note: deprecated route name \"" + fields[3] + ' ' + fields[4] \ + + "\" -> canonical name \"" + r2.readable_name() + "\" in line: " + line) + if r1 is None or r2 is None: + (lookup1, invchar) = no_control_chars(lookup1) + (lookup2, invchar) = no_control_chars(lookup2) + (line, invchar) = no_control_chars(line) + if r1 == r2: + self.log_entries.append("Unknown region/highway combos " + lookup1 + " and " + lookup2 + " in line: " + line) + else: + log_entry = "Unknown region/highway combo " + log_entry += lookup1 if r1 is None else lookup2 + log_entry += " in line: " + line + self.log_entries.append(log_entry) + if invchar: + self.log_entries[-1] += " [contains invalid character(s)]" + continue + if r1.con_route != r2.con_route: + self.log_entries.append(lookup1 + " and " + lookup2 + " not in same connected route in line: " + line) + continue + if r1.system.devel(): + self.log_entries.append("Ignoring line matching highway in system in development: " + line) + continue + # r1 and r2 are route matches, and we need to find + # waypoint indices, ignoring case and leading + # '+' or '*' when matching + list_label_1 = fields[2].lstrip("+*").upper() + list_label_2 = fields[5].lstrip("+*").upper() + # look for point indices for labels, first in pri_label_hash + # and then if not found, in alt_label_hash + try: + index1 = r1.pri_label_hash[list_label_1] + except KeyError: + try: + index1 = r1.alt_label_hash[list_label_1] + except KeyError: + index1 = None + try: + index2 = r2.pri_label_hash[list_label_2] + except KeyError: + try: + index2 = r2.alt_label_hash[list_label_2] + except KeyError: + index2 = None + # if we did not find matches for both labels... + if index1 is None or index2 is None: + (list_label_1, invchar) = no_control_chars(list_label_1) + (list_label_2, invchar) = no_control_chars(list_label_2) + (line, invchar) = no_control_chars(line) + if index1 is None and index2 is None: + self.log_entries.append("Waypoint labels " + list_label_1 + " and " + list_label_2 + " not found in line: " + line) + else: + log_entry = "Waypoint " + if index1 is None: + log_entry += lookup1 + ' ' + list_label_1 + else: + log_entry += lookup2 + ' ' + list_label_2 + log_entry += " not found in line: " + line + self.log_entries.append(log_entry) + if invchar: + self.log_entries[-1] += " [contains invalid character(s)]" + continue + # are either of the labels used duplicates? + duplicate = False + if list_label_1 in r1.duplicate_labels: + self.log_entries.append(r1.region + ": duplicate label " + list_label_1 + " in " + r1.root \ + + ". Please report this error in the TravelMapping forum. Unable to parse line: " + line) + duplicate = True + if list_label_2 in r2.duplicate_labels: + self.log_entries.append(r2.region + ": duplicate label " + list_label_2 + " in " + r2.root \ + + ". Please report this error in the TravelMapping forum. Unable to parse line: " + line) + duplicate = True + if duplicate: + continue + # if both region/route combos point to the same chopped route... + if r1 == r2: + # if both labels reference the same waypoint... + if index1 == index2: + self.log_entries.append("Equivalent waypoint labels mark zero distance traveled in line: " + line) + continue + if index1 <= index2: + r1.store_traveled_segments(self, index1, index2) + else: + r1.store_traveled_segments(self, index2, index1) + else: + if r1.rootOrder > r2.rootOrder: + (r1, r2) = (r2, r1) + (index1, index2) = (index2, index1) + # mark the beginning chopped route from index1 to its end + if r1.is_reversed: + r1.store_traveled_segments(self, 0, index1) + else: + r1.store_traveled_segments(self, index1, len(r1.segment_list)) + # mark the ending chopped route from its beginning to index2 + if r2.is_reversed: + r2.store_traveled_segments(self, index2, len(r2.segment_list)) + else: + r2.store_traveled_segments(self, 0, index2) + # mark any intermediate chopped routes in their entirety. + for r in range(r1.rootOrder+1, r2.rootOrder): + r1.con_route.roots[r].store_traveled_segments(self, 0, len(r1.con_route.roots[r].segment_list)) + # both labels are valid; mark in use & proceed + r1.system.listnamesinuse.add(lookup1) + r1.system.unusedaltroutenames.discard(lookup1) + r1.labels_in_use.add(list_label_1) + r2.system.listnamesinuse.add(lookup2) + r2.system.unusedaltroutenames.discard(lookup2) + r2.labels_in_use.add(list_label_2) + list_entries += 1 + else: + self.log_entries.append("Incorrect format line (4 or 6 fields expected, found " + \ + str(len(fields)) +"): " + line) self.log_entries.append("Processed " + str(list_entries) + \ - " good lines marking " +str(len(self.clinched_segments)) + \ - " segments traveled.") + " good lines marking " +str(len(self.clinched_segments)) + \ + " segments traveled.") # additional setup for later stats processing # a place to track this user's total mileage per region, # but only active+preview and active only (since devel @@ -1333,10 +1555,12 @@ class DatacheckEntry: -----------------------+-------------------------------------------- BAD_ANGLE | BUS_WITH_I | + DISCONNECTED_ROUTE | adjacent root's expected connection point DUPLICATE_COORDS | coordinate pair DUPLICATE_LABEL | HIDDEN_JUNCTION | number of incident edges in TM master graph HIDDEN_TERMINUS | + INTERSTATE_NO_HYPHEN | INVALID_FINAL_CHAR | final character in label INVALID_FIRST_CHAR | first character in label other than * LABEL_INVALID_CHAR | @@ -1355,7 +1579,7 @@ class DatacheckEntry: NONTERMINAL_UNDERSCORE | OUT_OF_BOUNDS | coordinate pair SHARP_ANGLE | angle in degrees - US_BANNER | + US_LETTER | VISIBLE_DISTANCE | distance in miles VISIBLE_HIDDEN_COLOC | hidden point at same coordinates @@ -2312,7 +2536,7 @@ def __init__(self,filename,descr,vertices,edges,travelers,format,category): " bytes in countries.csv line " + line) countries.append(fields) -all_regions = [] +all_regions = {} try: file = open(args.highwaydatapath+"/regions.csv", "rt",encoding='utf-8') except OSError as e: @@ -2354,11 +2578,11 @@ def __init__(self,filename,descr,vertices,edges,travelers,format,category): if len(fields[4].encode('utf-8')) > DBFieldLength.regiontype: el.add_error("Region type > " + str(DBFieldLength.regiontype) + " bytes in regions.csv line " + line) - all_regions.append(fields) + all_regions[fields[0]] = fields # Create a list of HighwaySystem objects, one per system in systems.csv file highway_systems = [] -print(et.et() + "Reading systems list in " + args.highwaydatapath+"/"+args.systemsfile + ". ",flush=True) +print(et.et() + "Reading systems list in " + args.highwaydatapath+"/"+args.systemsfile + ".",flush=True) try: file = open(args.highwaydatapath+"/"+args.systemsfile, "rt",encoding='utf-8') except OSError as e: @@ -2426,78 +2650,15 @@ def __init__(self,filename,descr,vertices,edges,travelers,format,category): # list for datacheck errors that we will need later datacheckerrors = [] -# check for duplicate root entries among Route and ConnectedRoute -# data in all highway systems -print(et.et() + "Checking for duplicate list names in routes, roots in routes and connected routes.",flush=True) -roots = set() -list_names = set() -duplicate_list_names = set() -for h in highway_systems: - for r in h.route_list: - if r.root in roots: - el.add_error("Duplicate root in route lists: " + r.root) - else: - roots.add(r.root) - list_name = r.region + ' ' + r.list_entry_name() - if list_name in list_names: - duplicate_list_names.add(list_name) - else: - list_names.add(list_name) - -con_roots = set() -for h in highway_systems: - for cr in h.con_route_list: - for r in cr.roots: - if r.root in con_roots: - el.add_error("Duplicate root in con_route lists: " + r.root) - else: - con_roots.add(r.root) - -# Make sure every route was listed as a part of some connected route -if len(roots) == len(con_roots): - print("Check passed: same number of routes as connected route roots. " + str(len(roots))) -else: - el.add_error("Check FAILED: " + str(len(roots)) + " routes != " + str(len(con_roots)) + " connected route roots.") - roots = roots - con_roots - # there will be some leftovers, let's look up their routes to make - # an error report entry (not worried about efficiency as there would - # only be a few in reasonable cases) - num_found = 0 - for h in highway_systems: - for r in h.route_list: - for lr in roots: - if lr == r.root: - el.add_error("route " + lr + " not matched by any connected route root.") - num_found += 1 - break - print("Added " + str(num_found) + " ROUTE_NOT_IN_CONNECTED error entries.") - -# report any duplicate list names as errors -if len(duplicate_list_names) > 0: - print("Found " + str(len(duplicate_list_names)) + " DUPLICATE_LIST_NAME case(s).") - for d in duplicate_list_names: - el.add_error("Duplicate list name: " + d) -else: - print("No duplicate list names found.") - -# write file mapping CHM datacheck route lists to root (commented out, -# unlikely needed now) -#print(et.et() + "Writing CHM datacheck to TravelMapping route pairings.") -#file = open(args.csvstatfilepath + "/routepairings.csv","wt") -#for h in highway_systems: -# for r in h.route_list: -# file.write(r.region + " " + r.list_entry_name() + ";" + r.root + "\n") -#file.close() - # For tracking whether any .wpt files are in the directory tree # that do not have a .csv file entry that causes them to be # read into the data print(et.et() + "Finding all .wpt files. ",end="",flush=True) -all_wpt_files = [] +all_wpt_files = set() for dir, sub, files in os.walk(args.highwaydatapath+"/hwy_data"): for file in files: if file.endswith('.wpt') and '_boundaries' not in dir: - all_wpt_files.append(dir+"/"+file) + all_wpt_files.add(dir+"/"+file) print(str(len(all_wpt_files)) + " files found.") # For finding colocated Waypoints and concurrent segments, we have @@ -2516,7 +2677,6 @@ def read_wpts_for_highway_system(h): all_wpt_files.remove(wpt_path) r.read_wpt(all_waypoints,all_waypoints_lock,datacheckerrors, el,args.highwaydatapath+"/hwy_data") - print(".", end="",flush=True) #print(str(r)) #r.print_route() print("!", flush=True) @@ -2718,14 +2878,58 @@ def run(self): print(".", end="", flush=True) print() -# Create hash table for faster lookup of routes by list file name -print(et.et() + "Creating route hash table for list processing:",flush=True) -route_hash = dict() +print(et.et() + "Processing waypoint labels and checking for unconnected chopped routes.",flush=True) for h in highway_systems: for r in h.route_list: - route_hash[(r.region + ' ' + r.list_entry_name()).lower()] = r - for a in r.alt_route_names: - route_hash[(r.region + ' ' + a).lower()] = r + # check for unconnected chopped routes + if r.con_route is None: + el.add_error(r.system.systemname + ".csv: root " + r.root + " not matched by any connected route root.") + + # check for mismatched route endpoints within connected routes + q = r.con_route.roots[r.rootOrder-1] + if r.rootOrder > 0 and len(q.point_list) > 1 and len(r.point_list) > 1 and not r.con_beg().same_coords(q.con_end()): + if q.con_beg().same_coords(r.con_beg()): + q.is_reversed = True + elif q.con_end().same_coords(r.con_end()): + r.is_reversed = True + elif q.con_beg().same_coords(r.con_end()): + q.is_reversed = True + r.is_reversed = True + else: + datacheckerrors.append(DatacheckEntry(r, [r.con_beg().label], + "DISCONNECTED_ROUTE", q.root + '@' + q.con_end().label)) + datacheckerrors.append(DatacheckEntry(q, [q.con_end().label], + "DISCONNECTED_ROUTE", r.root + '@' + r.con_beg().label)) + + # create label hashes and check for duplicates + index = 0 + for w in r.point_list: + # ignore case and leading '+' or '*' + upper_label = w.label.lstrip('+*').upper() + # if primary label not duplicated, add to r.pri_label_hash + if upper_label in r.alt_label_hash: + datacheckerrors.append(DatacheckEntry(r, [upper_label], "DUPLICATE_LABEL")) + r.duplicate_labels.add(upper_label) + elif upper_label in r.pri_label_hash: + datacheckerrors.append(DatacheckEntry(r, [upper_label], "DUPLICATE_LABEL")) + r.duplicate_labels.add(upper_label) + else: + r.pri_label_hash[upper_label] = index + for a in range(len(w.alt_labels)): + # create canonical AltLabels + w.alt_labels[a] = w.alt_labels[a].lstrip('+*').upper() + # populate unused set + r.unused_alt_labels.add(w.alt_labels[a]) + # create label->index hashes and check if AltLabels duplicated + if w.alt_labels[a] in r.pri_label_hash: + datacheckerrors.append(DatacheckEntry(r, [w.alt_labels[a]], "DUPLICATE_LABEL")) + r.duplicate_labels.add(w.alt_labels[a]) + elif w.alt_labels[a] in r.alt_label_hash: + datacheckerrors.append(DatacheckEntry(r, [w.alt_labels[a]], "DUPLICATE_LABEL")) + r.duplicate_labels.add(w.alt_labels[a]) + else: + r.alt_label_hash[w.alt_labels[a]] = index + index += 1 # Create a list of TravelerList objects, one per person traveler_lists = [] @@ -2734,7 +2938,7 @@ def run(self): for t in traveler_ids: if t.endswith('.list'): print(t + " ",end="",flush=True) - traveler_lists.append(TravelerList(t,route_hash,el,args.userlistfilepath)) + traveler_lists.append(TravelerList(t,el,args.userlistfilepath)) print('\n' + et.et() + "Processed " + str(len(traveler_lists)) + " traveler list files.") traveler_lists.sort(key=lambda TravelerList: TravelerList.traveler_name) # assign traveler numbers @@ -2743,10 +2947,20 @@ def run(self): t.traveler_num = travnum travnum += 1 +print(et.et() + "Clearing route & label hash tables.",flush=True) +del Route.root_hash +del Route.pri_list_hash +del Route.alt_list_hash +for h in highway_systems: + for r in h.route_list: + del r.pri_label_hash + del r.alt_label_hash + del r.duplicate_labels + # Read updates.csv file, just keep in the fields array for now since we're # just going to drop this into the DB later anyway updates = [] -print(et.et() + "Reading updates file. ",end="",flush=True) +print(et.et() + "Reading updates file.",end="",flush=True) with open(args.highwaydatapath+"/updates.csv", "rt", encoding='UTF-8') as file: lines = file.readlines() file.close() @@ -2784,7 +2998,7 @@ def run(self): # array for now since we're just going to drop this into the DB later # anyway systemupdates = [] -print(et.et() + "Reading systemupdates file. ",end="",flush=True) +print(et.et() + "Reading systemupdates file.",end="",flush=True) with open(args.highwaydatapath+"/systemupdates.csv", "rt", encoding='UTF-8') as file: lines = file.readlines() file.close() @@ -2818,42 +3032,61 @@ def run(self): systemupdates.append(fields) print("") -# write log file for points in use -- might be more useful in the DB later, -# or maybe in another format -print(et.et() + "Writing points in use log.") -inusefile = open(args.logfilepath+'/pointsinuse.log','w',encoding='UTF-8') -inusefile.write("Log file created at: " + str(datetime.datetime.now()) + "\n") -for h in highway_systems: - for r in h.route_list: - if len(r.labels_in_use) > 0: - inusefile.write(r.root + "(" + str(len(r.point_list)) + "):") - for label in sorted(r.labels_in_use): - inusefile.write(" " + label) - inusefile.write("\n") - r.labels_in_use = None -inusefile.close() - -# write log file for alt labels not in use -print(et.et() + "Writing unused alt labels log.") +print(et.et() + "Writing pointsinuse.log, unusedaltlabels.log, listnamesinuse.log and unusedaltroutenames.log",flush=True) total_unused_alt_labels = 0 +total_unusedaltroutenames = 0 unused_alt_labels = [] +piufile = open(args.logfilepath+'/pointsinuse.log','w',encoding='UTF-8') +lniufile = open(args.logfilepath+'/listnamesinuse.log','w',encoding='UTF-8') +uarnfile = open(args.logfilepath+'/unusedaltroutenames.log','w',encoding='UTF-8') +piufile.write("Log file created at: " + str(datetime.datetime.now()) + "\n") +lniufile.write("Log file created at: " + str(datetime.datetime.now()) + "\n") +uarnfile.write("Log file created at: " + str(datetime.datetime.now()) + "\n") for h in highway_systems: for r in h.route_list: + # labelsinuse.log line + if len(r.labels_in_use) > 0: + piufile.write(r.root + "(" + str(len(r.point_list)) + "):") + for label in sorted(r.labels_in_use): + piufile.write(" " + label) + piufile.write("\n") + del r.labels_in_use + # unusedaltlabels.log lines, to be sorted by root later if len(r.unused_alt_labels) > 0: total_unused_alt_labels += len(r.unused_alt_labels) ual_entry = r.root + "(" + str(len(r.unused_alt_labels)) + "):" for label in sorted(r.unused_alt_labels): ual_entry += " " + label - r.unused_alt_labels = None unused_alt_labels.append(ual_entry) + del r.unused_alt_labels + #listnamesinuse.log line + if len(h.listnamesinuse) > 0: + lniufile.write(h.systemname + '(' + str(len(h.route_list)) + "):") + for list_name in sorted(h.listnamesinuse): + lniufile.write(" \"" + list_name + '"') + lniufile.write('\n') + del h.listnamesinuse + # unusedaltroutenames.log line + if len(h.unusedaltroutenames) > 0: + total_unusedaltroutenames += len(h.unusedaltroutenames) + uarnfile.write(h.systemname + '(' + str(len(h.unusedaltroutenames)) + "):") + for list_name in sorted(h.unusedaltroutenames): + uarnfile.write(" \"" + list_name + '"') + uarnfile.write('\n') + del h.unusedaltroutenames +piufile.close() +lniufile.close() +uarnfile.write("Total: " + str(total_unusedaltroutenames) + '\n') +uarnfile.close() +# sort lines and write unusedaltlabels.log unused_alt_labels.sort() -unusedfile = open(args.logfilepath+'/unusedaltlabels.log','w',encoding='UTF-8') -unusedfile.write("Log file created at: " + str(datetime.datetime.now()) + "\n") +ualfile = open(args.logfilepath+'/unusedaltlabels.log','w',encoding='UTF-8') +ualfile.write("Log file created at: " + str(datetime.datetime.now()) + "\n") for ual_entry in unused_alt_labels: - unusedfile.write(ual_entry + "\n") + ualfile.write(ual_entry + "\n") unused_alt_labels = None -unusedfile.write("Total: " + str(total_unused_alt_labels) + "\n") -unusedfile.close() +ualfile.write("Total: " + str(total_unused_alt_labels) + "\n") +ualfile.close() # concurrency detection -- will augment our structure with list of concurrent @@ -3061,13 +3294,11 @@ def run(self): hdstatsfile.write(region + ": " + "{0:.2f}".format(h.mileage_by_region[region]) + " mi\n") hdstatsfile.write("System " + h.systemname + " by route:\n") for cr in h.con_route_list: - con_total_miles = 0.0 to_write = "" for r in cr.roots: to_write += " " + r.readable_name() + ": " + "{0:.2f}".format(r.mileage) + " mi\n" - con_total_miles += r.mileage - cr.mileage = con_total_miles - hdstatsfile.write(cr.readable_name() + ": " + "{0:.2f}".format(con_total_miles) + " mi") + cr.mileage += r.mileage + hdstatsfile.write(cr.readable_name() + ": " + "{0:.2f}".format(cr.mileage) + " mi") if len(cr.roots) == 1: hdstatsfile.write(" (" + cr.roots[0].readable_name() + " only)\n") else: @@ -3170,7 +3401,6 @@ def run(self): con_routes_clinched = 0 t.log_entries.append("System " + h.systemname + " by route (traveled routes only):") for cr in h.con_route_list: - con_total_miles = 0.0 con_clinched_miles = 0.0 to_write = "" for r in cr.roots: @@ -3187,18 +3417,17 @@ def run(self): con_clinched_miles += miles to_write += " " + r.readable_name() + ": " + \ format_clinched_mi(miles,r.mileage) + "\n" - con_total_miles += r.mileage if con_clinched_miles > 0: system_con_dict[cr] = con_clinched_miles clinched = '0' - if con_clinched_miles == con_total_miles: + if con_clinched_miles == cr.mileage: con_routes_clinched += 1 clinched = '1' ccr_values.append("('" + cr.roots[0].root + "','" + t.traveler_name + "','" + str(con_clinched_miles) + "','" + clinched + "')") t.log_entries.append(cr.readable_name() + ": " + \ - format_clinched_mi(con_clinched_miles,con_total_miles)) + format_clinched_mi(con_clinched_miles,cr.mileage)) if len(cr.roots) == 1: t.log_entries.append(" (" + cr.roots[0].readable_name() + " only)") else: @@ -3310,12 +3539,13 @@ def run(self): lines.pop(0) # ignore header line datacheckfps = [] -datacheck_always_error = [ 'BAD_ANGLE', 'DUPLICATE_LABEL', 'HIDDEN_TERMINUS', +datacheck_always_error = [ 'BAD_ANGLE', 'DISCONNECTED_ROUTE', 'DUPLICATE_LABEL', + 'HIDDEN_TERMINUS', 'INTERSTATE_NO_HYPHEN', 'INVALID_FINAL_CHAR', 'INVALID_FIRST_CHAR', 'LABEL_INVALID_CHAR', 'LABEL_PARENS', 'LABEL_SLASHES', 'LABEL_TOO_LONG', 'LABEL_UNDERSCORES', 'LONG_UNDERSCORE', 'MALFORMED_LAT', 'MALFORMED_LON', 'MALFORMED_URL', - 'NONTERMINAL_UNDERSCORE' ] + 'NONTERMINAL_UNDERSCORE', 'US_LETTER' ] for line in lines: line=line.strip() if len(line) == 0: @@ -3420,7 +3650,7 @@ def run(self): # We will create graph data and a graph file for each region that includes # any active or preview systems - for r in all_regions: + for r in all_regions.values(): region_code = r[0] if region_code not in active_preview_mileage_by_region: continue @@ -3526,9 +3756,9 @@ def run(self): print(fields[1] + ' ', end="", flush=True) region_list = [] selected_regions = fields[2].split(",") - for r in all_regions: - if r[0] in selected_regions and r[0] in active_preview_mileage_by_region: - region_list.append(r[0]) + for r in all_regions.keys(): + if r in selected_regions and r in active_preview_mileage_by_region: + region_list.append(r) graph_data.write_subgraphs_tmg(graph_list, args.graphfilepath + "/", fields[1], fields[0], "multiregion", region_list, None, None, all_waypoints) @@ -3543,7 +3773,7 @@ def run(self): print(et.et() + "Creating country graphs.", flush=True) for c in countries: region_list = [] - for r in all_regions: + for r in all_regions.values(): # does it match this country and have routes? if c[0] == r[2] and r[0] in active_preview_mileage_by_region: region_list.append(r[0]) @@ -3564,7 +3794,7 @@ def run(self): print(et.et() + "Creating continent graphs.", flush=True) for c in continents: region_list = [] - for r in all_regions: + for r in all_regions.values(): # does it match this continent and have routes? if c[0] == r[3] and r[0] in active_preview_mileage_by_region: region_list.append(r[0]) @@ -3583,9 +3813,8 @@ def run(self): # perform most datachecks here (list initialized above) for h in highway_systems: print(".",end="",flush=True) + usa_flag = h.country == "USA" for r in h.route_list: - # set to be used per-route to find label duplicates - all_route_labels = set() # set of tuples to be used for finding duplicate coordinates coords_used = set() @@ -3600,24 +3829,9 @@ def run(self): if r.point_list[0].is_hidden: datacheckerrors.append(DatacheckEntry(r,[r.point_list[0].label],'HIDDEN_TERMINUS')) if r.point_list[-1].is_hidden: - datacheckerrors.append(DatacheckEntry(r,[r.point_list[len(r.point_list)-1].label],'HIDDEN_TERMINUS')) + datacheckerrors.append(DatacheckEntry(r,[r.point_list[-1].label],'HIDDEN_TERMINUS')) for w in r.point_list: - # duplicate labels - # first, check primary label - lower_label = w.label.lower().strip("+*") - if lower_label in all_route_labels: - datacheckerrors.append(DatacheckEntry(r,[lower_label],"DUPLICATE_LABEL")) - else: - all_route_labels.add(lower_label) - # then check alt labels - for label in w.alt_labels: - lower_label = label.lower().strip("+*") - if lower_label in all_route_labels: - datacheckerrors.append(DatacheckEntry(r,[lower_label],"DUPLICATE_LABEL")) - else: - all_route_labels.add(lower_label) - # out-of-bounds coords if w.lat > 90 or w.lat < -90 or w.lng > 180 or w.lng < -180: datacheckerrors.append(DatacheckEntry(r,[w.label],'OUT_OF_BOUNDS', @@ -3638,6 +3852,13 @@ def run(self): else: coords_used.add(latlng) + # look for labels with invalid characters + if not re.fullmatch('\+?\*?[a-zA-Z0-9()/_\-\.]+', w.label): + datacheckerrors.append(DatacheckEntry(r,[w.label],'LABEL_INVALID_CHAR')) + for a in w.alt_labels: + if not re.fullmatch('\+?\*?[a-zA-Z0-9()/_\-\.]+', a): + datacheckerrors.append(DatacheckEntry(r,[a],'LABEL_INVALID_CHAR')) + # visible distance update, and last segment length check if prev_w is not None: last_distance = w.distance_to(prev_w) @@ -3662,15 +3883,7 @@ def run(self): visible_distance = 0.0 # looking for the route within the label - #match_start = w.label.find(r.route) - #if match_start >= 0: - # we have a potential match, just need to make sure if the route - # name ends with a number that the matched substring isn't followed - # by more numbers (e.g., NY50 is an OK label in NY5) - # if len(r.route) + match_start == len(w.label) or \ - # not w.label[len(r.route) + match_start].isdigit(): # partially complete "references own route" -- too many FP - #or re.fullmatch('.*/'+r.route+'.*',w.label[w.label) : # first check for number match after a slash, if there is one selfref_found = False if '/' in w.label and r.route[-1].isdigit(): @@ -3685,7 +3898,6 @@ def run(self): selfref_found = True if '_' in w.label[w.label.index('/')+1:] and w.label[w.label.index('/')+1:w.label.rindex('_')] == r.route: selfref_found = True - # now the remaining checks if selfref_found or r.route+r.banner == w.label or re.fullmatch(r.route+r.banner+'[_/].*',w.label): datacheckerrors.append(DatacheckEntry(r,[w.label],'LABEL_SELFREF')) @@ -3712,16 +3924,9 @@ def run(self): or (left_count == 1 and w.label.index('(') > w.label.index(')')): datacheckerrors.append(DatacheckEntry(r,[w.label],'LABEL_PARENS')) - # look for labels with invalid characters - if not re.fullmatch('\*?[a-zA-Z0-9()/_\-\.]+', w.label): - datacheckerrors.append(DatacheckEntry(r,[w.label],'LABEL_INVALID_CHAR')) - for a in w.alt_labels: - if not re.fullmatch('\+?\*?[a-zA-Z0-9()/_\-\.]+', a): - datacheckerrors.append(DatacheckEntry(r,[a],'LABEL_INVALID_CHAR')) - # look for labels with invalid first or last character index = 0 - while w.label[index] == '*': + while index < len(w.label) and w.label[index] == '*': index += 1 if index < len(w.label) and w.label[index] in "_/(": datacheckerrors.append(DatacheckEntry(r,[w.label],'INVALID_FIRST_CHAR', w.label[index])) @@ -3733,21 +3938,48 @@ def run(self): w.label.index('/') > w.label.index('_'): datacheckerrors.append(DatacheckEntry(r,[w.label],'NONTERMINAL_UNDERSCORE')) - # look for I-xx with Bus instead of BL or BS - if re.fullmatch('I\-[0-9]*Bus', w.label): - datacheckerrors.append(DatacheckEntry(r,[w.label],'BUS_WITH_I')) - # look for labels that look like hidden waypoints but # which aren't hidden if re.fullmatch('X[0-9][0-9][0-9][0-9][0-9][0-9]', w.label): datacheckerrors.append(DatacheckEntry(r,[w.label],'LABEL_LOOKS_HIDDEN')) - # look for USxxxA but not USxxxAlt, B/Bus (others?) - ##if re.fullmatch('US[0-9]+A.*', w.label) and not re.fullmatch('US[0-9]+Alt.*', w.label) or \ - ## re.fullmatch('US[0-9]+B.*', w.label) and \ - ## not (re.fullmatch('US[0-9]+Bus.*', w.label) or re.fullmatch('US[0-9]+Byp.*', w.label)): - ## datacheckerrors.append(DatacheckEntry(r,[w.label],'US_BANNER')) - + # lacks generic highway type + if re.fullmatch('^\*?[Oo][lL][dD][0-9].*', w.label): + datacheckerrors.append(DatacheckEntry(r,[w.label],'LACKS_GENERIC')) + + # USA-only datachecks + if usa_flag and len(w.label) >= 2: + + # look for I-xx with Bus instead of BL or BS + if re.fullmatch('\*?I\-[0-9]+[EeWwCcNnSs]?[Bb][Uu][Ss].*', w.label): + datacheckerrors.append(DatacheckEntry(r,[w.label],'BUS_WITH_I')) + + # look for Ixx without hyphen + c = 1 if w.label[0] == '*' else 0 + if w.label[c:c+2] == "To": + c += 2; + if len(w.label) >= c+2 and w.label[c] == 'I' and w.label[c+1].isdigit(): + datacheckerrors.append(DatacheckEntry(r,[w.label],'INTERSTATE_NO_HYPHEN')) + + # look for USxxxA but not USxxxAlt, B/Bus/Byp + # Eric's paraphrase of Jim's original criteria + # if re.fullmatch('\*?US[0-9]+[AB].*', w.label) and not re.fullmatch('\*?US[0-9]+Alt.*|\*?US[0-9]+Bus.*|\*?US[0-9]+Byp.*', w.label): + # Instead, let's cast a narrower net (optimized for speed, just because): + try: + c = 3 if w.label[0] == '*' else 2 + if w.label[c-2] == 'U' and w.label[c-1] == 'S' and w.label[c].isdigit(): + while w.label[c].isdigit(): + c += 1 + if w.label[c] in 'AB': + c += 1 + if c == len(w.label) or w.label[c] in '/_(': + datacheckerrors.append(DatacheckEntry(r,[w.label],'US_LETTER')) + # is it followed by a city abbrev? + elif w.label[c].isupper() and w.label[c+1].islower() and w.label[c+2].islower() \ + and (c+3 == len(w.label) or w.label[c+3] in '/_('): + datacheckerrors.append(DatacheckEntry(r,[w.label],'US_LETTER')) + except IndexError: + pass prev_w = w # angle check is easier with a traditional for loop and array indices @@ -3770,7 +4002,6 @@ def run(self): datacheckerrors.append(DatacheckEntry(r,labels,'SHARP_ANGLE', "{0:.2f}".format(angle))) print("!", flush=True) -print(et.et() + "Found " + str(len(datacheckerrors)) + " datacheck errors.") datacheckerrors.sort(key=lambda DatacheckEntry: str(DatacheckEntry)) @@ -3778,7 +4009,6 @@ def run(self): print(et.et() + "Marking datacheck false positives.",end="",flush=True) fpfile = open(args.logfilepath+'/nearmatchfps.log','w',encoding='utf-8') fpfile.write("Log file created at: " + str(datetime.datetime.now()) + "\n") -toremove = [] counter = 0 fpcount = 0 for d in datacheckerrors: @@ -3799,7 +4029,7 @@ def run(self): fpfile.write("CHANGETO: " + fp[0] + ';' + fp[1] + ';' + fp[2] + ';' + fp[3] + ';' + fp[4] + ';' + d.info + '\n') fpfile.close() print("!", flush=True) -print(et.et() + "Matched " + str(fpcount) + " FP entries.", flush=True) +print(et.et() + "Found " + str(len(datacheckerrors)) + " datacheck errors and matched " + str(fpcount) + " FP entries.", flush=True) # write log of unmatched false positives from the datacheckfps.csv print(et.et() + "Writing log of unmatched datacheck FP entries.", flush=True) @@ -3898,7 +4128,7 @@ def run(self): '), PRIMARY KEY(code), FOREIGN KEY (country) REFERENCES countries(code), FOREIGN KEY (continent) REFERENCES continents(code));\n') sqlfile.write('INSERT INTO regions VALUES\n') first = True - for r in all_regions: + for r in all_regions.values(): if not first: sqlfile.write(",") first = False