From 2c75863fe6dafd6347a11c2e1fcbd17920cebe80 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Sat, 25 Apr 2020 02:36:25 -0400 Subject: [PATCH 01/34] wpt whitespace/blank line bugfix --- siteupdate/cplusplus/classes/Route/read_wpt.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteupdate/cplusplus/classes/Route/read_wpt.cpp b/siteupdate/cplusplus/classes/Route/read_wpt.cpp index 1be5cadc..106b3dd0 100644 --- a/siteupdate/cplusplus/classes/Route/read_wpt.cpp +++ b/siteupdate/cplusplus/classes/Route/read_wpt.cpp @@ -39,7 +39,7 @@ void Route::read_wpt { // strip whitespace while (lines[l][0] == ' ' || lines[l][0] == '\t') lines[l]++; char * endchar = lines[l+1]-2; // -2 skips over the 0 inserted by strtok - if (*endchar == 0) endchar--; // skip back one more for CRLF cases FIXME what about lines followed by blank lines? + while (*endchar == 0) endchar--; // skip back more for CRLF cases, and lines followed by blank lines while (*endchar == ' ' || *endchar == '\t') { *endchar = 0; endchar--; From 14ea929383adef4b52e2d3e4d1666b369ee180cc Mon Sep 17 00:00:00 2001 From: eric bryant Date: Sat, 25 Apr 2020 02:57:29 -0400 Subject: [PATCH 02/34] no strtok_mtx when reading waypoints --- siteupdate/cplusplus/classes/Route/Route.h | 2 +- .../cplusplus/classes/Route/read_wpt.cpp | 19 ++++++++++++++----- .../cplusplus/classes/Waypoint/Waypoint.cpp | 15 ++++++++++----- .../cplusplus/classes/Waypoint/Waypoint.h | 2 +- siteupdate/cplusplus/siteupdate.cpp | 4 ++-- .../cplusplus/threads/ReadWptThread.cpp | 4 ++-- 6 files changed, 30 insertions(+), 16 deletions(-) diff --git a/siteupdate/cplusplus/classes/Route/Route.h b/siteupdate/cplusplus/classes/Route/Route.h index b1949da7..c67a1479 100644 --- a/siteupdate/cplusplus/classes/Route/Route.h +++ b/siteupdate/cplusplus/classes/Route/Route.h @@ -61,7 +61,7 @@ class Route Route(std::string &, HighwaySystem *, ErrorList &, std::unordered_map &); std::string str(); - void read_wpt(WaypointQuadtree *, ErrorList *, std::string, std::mutex *, DatacheckEntryList *, std::unordered_set *); + void read_wpt(WaypointQuadtree *, ErrorList *, std::string, DatacheckEntryList *, std::unordered_set *); void print_route(); HighwaySegment* find_segment_by_waypoints(Waypoint*, Waypoint*); std::string chopped_rtes_line(); diff --git a/siteupdate/cplusplus/classes/Route/read_wpt.cpp b/siteupdate/cplusplus/classes/Route/read_wpt.cpp index 106b3dd0..bc261744 100644 --- a/siteupdate/cplusplus/classes/Route/read_wpt.cpp +++ b/siteupdate/cplusplus/classes/Route/read_wpt.cpp @@ -1,5 +1,5 @@ void Route::read_wpt -( WaypointQuadtree *all_waypoints, ErrorList *el, std::string path, std::mutex *strtok_mtx, +( WaypointQuadtree *all_waypoints, ErrorList *el, std::string path, DatacheckEntryList *datacheckerrors, std::unordered_set *all_wpt_files ) { /* read data into the Route's waypoint list from a .wpt file */ @@ -23,9 +23,18 @@ void Route::read_wpt file.read(wptdata, wptdatasize); wptdata[wptdatasize] = 0; // add null terminator file.close(); - strtok_mtx->lock(); - for (char *token = strtok(wptdata, "\r\n"); token; token = strtok(0, "\r\n") ) lines.emplace_back(token); - strtok_mtx->unlock(); + + // split file into lines + size_t spn = 0; + for (char* c = wptdata; *c; c += spn) + { spn = strcspn(c, "\n\r"); + while (c[spn] == '\n' || c[spn] == '\r') + { c[spn] = 0; + spn++; + } + lines.emplace_back(c); + } + lines.push_back(wptdata+wptdatasize+1); // add a dummy "past-the-end" element to make lines[l+1]-2 work // set to be used per-route to find label duplicates std::unordered_set all_route_labels; @@ -45,7 +54,7 @@ void Route::read_wpt endchar--; } if (lines[l][0] == 0) continue; - Waypoint *w = new Waypoint(lines[l], this, strtok_mtx, datacheckerrors); + Waypoint *w = new Waypoint(lines[l], this, datacheckerrors); // deleted on termination of program, or immediately below if invalid bool malformed_url = w->lat == 0 && w->lng == 0; bool label_too_long = w->label_too_long(datacheckerrors); diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp index 890724bf..7b3bfe20 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp @@ -15,15 +15,20 @@ bool waypoint_simplification_sort(Waypoint *w1, Waypoint *w2) const double Waypoint::pi = 3.141592653589793238; -Waypoint::Waypoint(char *line, Route *rte, std::mutex *strtok_mtx, DatacheckEntryList *datacheckerrors) +Waypoint::Waypoint(char *line, Route *rte, DatacheckEntryList *datacheckerrors) { /* initialize object from a .wpt file line */ route = rte; // parse WPT line - strtok_mtx->lock(); - for (char *token = strtok(line, " "); token; token = strtok(0, " ")) - alt_labels.emplace_back(token); // get all tokens & put into label deque - strtok_mtx->unlock(); + size_t spn = 0; + for (char* c = line; *c; c += spn) + { spn = strcspn(c, " "); + while (c[spn] == ' ') + { c[spn] = 0; + spn++; + } + alt_labels.emplace_back(c); + } // We know alt_labels will have at least one element, because if the WPT line is // blank or contains only spaces, Route::read_wpt will not call this constructor. diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.h b/siteupdate/cplusplus/classes/Waypoint/Waypoint.h index 694c2c90..a419e8d3 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.h +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.h @@ -22,7 +22,7 @@ class Waypoint bool is_hidden; static const double pi; - Waypoint(char *, Route *, std::mutex *, DatacheckEntryList *); + Waypoint(char *, Route *, DatacheckEntryList *); std::string str(); std::string csv_line(unsigned int); diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index 90f97f55..cef22302 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -335,7 +335,7 @@ int main(int argc, char *argv[]) thread **thr = new thread*[args.numthreads]; for (unsigned int t = 0; t < args.numthreads; t++) thr[t] = new thread(ReadWptThread, t, &highway_systems, &hs_it, &list_mtx, args.highwaydatapath+"/hwy_data", - &el, &all_wpt_files, &all_waypoints, &strtok_mtx, datacheckerrors); + &el, &all_wpt_files, &all_waypoints, datacheckerrors); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); for (unsigned int t = 0; t < args.numthreads; t++) @@ -344,7 +344,7 @@ int main(int argc, char *argv[]) for (HighwaySystem* h : highway_systems) { std::cout << h->systemname << std::flush; for (std::list::iterator r = h->route_list.begin(); r != h->route_list.end(); r++) - { r->read_wpt(&all_waypoints, &el, args.highwaydatapath+"/hwy_data", &strtok_mtx, datacheckerrors, &all_wpt_files); + { r->read_wpt(&all_waypoints, &el, args.highwaydatapath+"/hwy_data", datacheckerrors, &all_wpt_files); } std::cout << "!" << std::endl; } diff --git a/siteupdate/cplusplus/threads/ReadWptThread.cpp b/siteupdate/cplusplus/threads/ReadWptThread.cpp index e4c00de8..712111f7 100644 --- a/siteupdate/cplusplus/threads/ReadWptThread.cpp +++ b/siteupdate/cplusplus/threads/ReadWptThread.cpp @@ -1,7 +1,7 @@ void ReadWptThread ( unsigned int id, std::list *hs_list, std::list::iterator *it, std::mutex *hs_mtx, std::string path, ErrorList *el, std::unordered_set *all_wpt_files, - WaypointQuadtree *all_waypoints, std::mutex *strtok_mtx, DatacheckEntryList *datacheckerrors + WaypointQuadtree *all_waypoints, DatacheckEntryList *datacheckerrors ) { //printf("Starting ReadWptThread %02i\n", id); fflush(stdout); while (*it != hs_list->end()) @@ -17,7 +17,7 @@ void ReadWptThread hs_mtx->unlock(); std::cout << h->systemname << std::flush; for (Route &r : h->route_list) - r.read_wpt(all_waypoints, el, path, strtok_mtx, datacheckerrors, all_wpt_files); + r.read_wpt(all_waypoints, el, path, datacheckerrors, all_wpt_files); std::cout << "!" << std::endl; } } From 34c407ac011a9f54c28499390d4cdab28f62fbad Mon Sep 17 00:00:00 2001 From: eric bryant Date: Sun, 26 Apr 2020 13:16:25 -0400 Subject: [PATCH 03/34] update comment: no more strtok --- siteupdate/cplusplus/classes/Route/read_wpt.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteupdate/cplusplus/classes/Route/read_wpt.cpp b/siteupdate/cplusplus/classes/Route/read_wpt.cpp index bc261744..433c025f 100644 --- a/siteupdate/cplusplus/classes/Route/read_wpt.cpp +++ b/siteupdate/cplusplus/classes/Route/read_wpt.cpp @@ -47,7 +47,7 @@ void Route::read_wpt for (unsigned int l = 0; l < lines.size()-1; l++) { // strip whitespace while (lines[l][0] == ' ' || lines[l][0] == '\t') lines[l]++; - char * endchar = lines[l+1]-2; // -2 skips over the 0 inserted by strtok + char * endchar = lines[l+1]-2; // -2 skips over the 0 inserted while splitting wptdata into lines while (*endchar == 0) endchar--; // skip back more for CRLF cases, and lines followed by blank lines while (*endchar == ' ' || *endchar == '\t') { *endchar = 0; From e0c21d110b1a4b8d556d935ac9a31f9552309025 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Mon, 27 Apr 2020 11:02:16 -0400 Subject: [PATCH 04/34] all_wpt_files: list -> set --- siteupdate/python-teresco/siteupdate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index d2cce465..5cb4cdb1 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2493,11 +2493,11 @@ def __init__(self,filename,descr,vertices,edges,travelers,format,category): # that do not have a .csv file entry that causes them to be # read into the data print(et.et() + "Finding all .wpt files. ",end="",flush=True) -all_wpt_files = [] +all_wpt_files = set() for dir, sub, files in os.walk(args.highwaydatapath+"/hwy_data"): for file in files: if file.endswith('.wpt') and '_boundaries' not in dir: - all_wpt_files.append(dir+"/"+file) + all_wpt_files.add(dir+"/"+file) print(str(len(all_wpt_files)) + " files found.") # For finding colocated Waypoints and concurrent segments, we have From 8c633d2dff6bb430bc79e4f84dd4cdbce986e3f5 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Mon, 27 Apr 2020 14:12:44 -0400 Subject: [PATCH 05/34] syntactic sugar --- siteupdate/python-teresco/siteupdate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 5cb4cdb1..9e764491 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -3600,7 +3600,7 @@ def run(self): if r.point_list[0].is_hidden: datacheckerrors.append(DatacheckEntry(r,[r.point_list[0].label],'HIDDEN_TERMINUS')) if r.point_list[-1].is_hidden: - datacheckerrors.append(DatacheckEntry(r,[r.point_list[len(r.point_list)-1].label],'HIDDEN_TERMINUS')) + datacheckerrors.append(DatacheckEntry(r,[r.point_list[-1].label],'HIDDEN_TERMINUS')) for w in r.point_list: # duplicate labels From 8bcc7644c5dd32821ce38fa042ef69c6d8700266 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Mon, 4 May 2020 01:10:13 -0400 Subject: [PATCH 06/34] refactor roots & list names; smash case * Refactor checks for root & list name uniqueness and Route/ConnectedRoute correspondence for improved efficiency & error reporting and more concise code (#316) * Perform fewer case conversions on AltLabels & AltRouteNames for increased speed Processing traveler list files (#319) * C++: clean up a couple classes' static member definitions --- .../classes/ConnectedRoute/ConnectedRoute.cpp | 23 ++- .../classes/ConnectedRoute/ConnectedRoute.h | 2 +- .../classes/GraphGeneration/HGEdge.cpp | 5 - .../classes/GraphGeneration/HGEdge.h | 6 +- .../classes/GraphGeneration/PlaceRadius.cpp | 16 +- .../cplusplus/classes/HighwaySystem.cpp | 8 +- siteupdate/cplusplus/classes/Route/Route.cpp | 31 ++- siteupdate/cplusplus/classes/Route/Route.h | 1 + .../cplusplus/classes/Route/read_wpt.cpp | 10 - .../classes/TravelerList/TravelerList.cpp | 34 ++-- .../cplusplus/classes/Waypoint/Waypoint.cpp | 19 +- .../cplusplus/classes/Waypoint/Waypoint.h | 1 - siteupdate/cplusplus/functions/lower.cpp | 4 +- siteupdate/cplusplus/functions/upper.cpp | 6 + siteupdate/cplusplus/siteupdate.cpp | 82 ++------ .../cplusplus/threads/ReadListThread.cpp | 4 +- siteupdate/python-teresco/siteupdate.py | 189 +++++++----------- 17 files changed, 179 insertions(+), 262 deletions(-) diff --git a/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.cpp b/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.cpp index 5e0ddd85..c043df6b 100644 --- a/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.cpp +++ b/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.cpp @@ -1,4 +1,4 @@ -ConnectedRoute::ConnectedRoute(std::string &line, HighwaySystem *sys, ErrorList &el, std::list &route_list) +ConnectedRoute::ConnectedRoute(std::string &line, HighwaySystem *sys, ErrorList &el) { mileage = 0; // parse chopped routes csv line @@ -29,19 +29,28 @@ ConnectedRoute::ConnectedRoute(std::string &line, HighwaySystem *sys, ErrorList el.add_error("groupname > " + std::to_string(DBFieldLength::city) + " bytes in " + system->systemname + "_con.csv line: " + line); // roots + lower(roots_str.data()); int rootOrder = 0; size_t l = 0; for (size_t r = 0; r != -1; l = r+1) { r = roots_str.find(',', l); - Route *root = route_by_root(roots_str.substr(l, r-l), route_list); - if (!root) el.add_error("Could not find Route matching ConnectedRoute root " + roots_str.substr(l, r-l) + - " in system " + system->systemname + '.'); - else { roots.push_back(root); + try { Route *root = Route::root_hash.at(roots_str.substr(l, r-l)); + roots.push_back(root); + if (root->con_route) + el.add_error("Duplicate root in " + sys->systemname + "_con.csv: " + root->root + + " already in " + root->con_route->system->systemname + "_con.csv"); + if (system != root->system) + el.add_error("System mismatch: chopped route " + root->root + " from " + root->system->systemname + + ".csv in connected route in " + system->systemname + "_con.csv"); root->con_route = this; // save order of route in connected route root->rootOrder = rootOrder; - } - rootOrder++; + rootOrder++; + } + catch (std::out_of_range& oor) + { el.add_error("Could not find Route matching ConnectedRoute root " + roots_str.substr(l, r-l) + + " in system " + system->systemname + '.'); + } } if (roots.size() < 1) el.add_error("No roots in " + system->systemname + "_con.csv line: " + line); } diff --git a/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.h b/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.h index c9d05cfe..9692f937 100644 --- a/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.h +++ b/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.h @@ -12,7 +12,7 @@ class ConnectedRoute double mileage; // will be computed for routes in active & preview systems - ConnectedRoute(std::string &, HighwaySystem *, ErrorList &, std::list &); + ConnectedRoute(std::string &, HighwaySystem *, ErrorList &); std::string connected_rtes_line(); std::string csv_line(); diff --git a/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.cpp b/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.cpp index 579d8e81..24158034 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.cpp +++ b/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.cpp @@ -1,8 +1,3 @@ -// constants for more human-readable format masks -const unsigned char HGEdge::simple = 1; -const unsigned char HGEdge::collapsed = 2; -const unsigned char HGEdge::traveled = 4; - HGEdge::HGEdge(HighwaySegment *s, HighwayGraph *graph) { // initial construction is based on a HighwaySegment s_written = 0; // simple diff --git a/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.h b/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.h index 7b6374e0..3f29ba3a 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.h +++ b/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.h @@ -12,9 +12,9 @@ class HGEdge unsigned char format; // constants for more human-readable format masks - static const unsigned char simple; - static const unsigned char collapsed; - static const unsigned char traveled; + static const unsigned char simple = 1; + static const unsigned char collapsed = 2; + static const unsigned char traveled = 4; HGEdge(HighwaySegment *, HighwayGraph *); HGEdge(HGVertex *, unsigned char); diff --git a/siteupdate/cplusplus/classes/GraphGeneration/PlaceRadius.cpp b/siteupdate/cplusplus/classes/GraphGeneration/PlaceRadius.cpp index 63a4a77e..78b522a3 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/PlaceRadius.cpp +++ b/siteupdate/cplusplus/classes/GraphGeneration/PlaceRadius.cpp @@ -10,10 +10,10 @@ bool PlaceRadius::contains_vertex(HGVertex *v) {return contains_vertex(v->lat, v bool PlaceRadius::contains_vertex(double vlat, double vlng) { /* return whether coordinates are within this area */ // convert to radians to compute distance - double rlat1 = lat * (Waypoint::pi/180); - double rlng1 = lng * (Waypoint::pi/180); - double rlat2 = vlat * (Waypoint::pi/180); - double rlng2 = vlng * (Waypoint::pi/180); + double rlat1 = lat * (pi/180); + double rlng1 = lng * (pi/180); + double rlat2 = vlat * (pi/180); + double rlng2 = vlng * (pi/180); /* original formula double ans = acos(cos(rlat1)*cos(rlng1)*cos(rlat2)*cos(rlng2) +\ @@ -48,10 +48,10 @@ std::unordered_set PlaceRadius::vertices(WaypointQuadtree *qt, Highwa // N/S sanity check: If lat is <= r/2 miles to the N or S pole, lngdelta calculation will fail. // In these cases, our place radius will span the entire "width" of the world, from -180 to +180 degrees. - if (90-fabs(lat)*(Waypoint::pi/180) <= r/7926.2) return v_search(qt, g, -180, +180); + if (90-fabs(lat)*(pi/180) <= r/7926.2) return v_search(qt, g, -180, +180); // width, in degrees longitude, of our bounding box for quadtree search - double lngdelta = acos((cos(r/3963.1) - pow(sin(lat*(Waypoint::pi/180)),2)) / pow(cos(lat*(Waypoint::pi/180)),2)) / (Waypoint::pi/180); + double lngdelta = acos((cos(r/3963.1) - pow(sin(lat*(pi/180)),2)) / pow(cos(lat*(pi/180)),2)) / (pi/180); double w_bound = lng-lngdelta; double e_bound = lng+lngdelta; @@ -92,8 +92,8 @@ std::unordered_set PlaceRadius::v_search(WaypointQuadtree *qt, Highwa // if we're not a terminal quadrant, we need to determine which // of our child quadrants we need to search and recurse into each else { //printf("DEBUG: recursive case, mid_lat=%.17g mid_lng=%.17g\n", qt->mid_lat, qt->mid_lng); fflush(stdout); - bool look_n = (lat + r/3963.1/(Waypoint::pi/180)) >= qt->mid_lat; - bool look_s = (lat - r/3963.1/(Waypoint::pi/180)) <= qt->mid_lat; + bool look_n = (lat + r/3963.1/(pi/180)) >= qt->mid_lat; + bool look_s = (lat - r/3963.1/(pi/180)) <= qt->mid_lat; bool look_e = e_bound >= qt->mid_lng; bool look_w = w_bound <= qt->mid_lng; //std::cout << "DEBUG: recursive case, " << look_n << " " << look_s << " " << look_e << " " << look_w << std::endl; diff --git a/siteupdate/cplusplus/classes/HighwaySystem.cpp b/siteupdate/cplusplus/classes/HighwaySystem.cpp index fbdd9054..0d6bc2b3 100644 --- a/siteupdate/cplusplus/classes/HighwaySystem.cpp +++ b/siteupdate/cplusplus/classes/HighwaySystem.cpp @@ -79,7 +79,8 @@ class HighwaySystem if (!file) el.add_error("Could not open "+path+"/"+systemname+".csv"); else { getline(file, line); // ignore header line while(getline(file, line)) - { if (line.back() == 0x0D) line.erase(line.end()-1); // trim DOS newlines + { // trim DOS newlines & trailing whitespace + while ( strchr("\r\t ", line.back()) ) line.pop_back(); if (line.empty()) continue; route_list.emplace_back(line, this, el, region_hash); if (route_list.back().root.empty()) @@ -95,9 +96,10 @@ class HighwaySystem if (!file) el.add_error("Could not open "+path+"/"+systemname+"_con.csv"); else { getline(file, line); // ignore header line while(getline(file, line)) - { if (line.back() == 0x0D) line.erase(line.end()-1); // trim DOS newlines + { // trim DOS newlines & trailing whitespace + while ( strchr("\r\t ", line.back()) ) line.pop_back(); if (line.empty()) continue; - con_route_list.emplace_back(line, this, el, route_list); + con_route_list.emplace_back(line, this, el); } } file.close(); diff --git a/siteupdate/cplusplus/classes/Route/Route.cpp b/siteupdate/cplusplus/classes/Route/Route.cpp index 6c104d27..4cbf193c 100644 --- a/siteupdate/cplusplus/classes/Route/Route.cpp +++ b/siteupdate/cplusplus/classes/Route/Route.cpp @@ -1,3 +1,4 @@ +std::unordered_map Route::root_hash, Route::list_hash; std::mutex Route::awf_mtx; Route::Route(std::string &line, HighwaySystem *sys, ErrorList &el, std::unordered_map ®ion_hash) @@ -52,12 +53,28 @@ Route::Route(std::string &line, HighwaySystem *sys, ErrorList &el, std::unordere if (root.size() > DBFieldLength::root) el.add_error("Root > " + std::to_string(DBFieldLength::root) + " bytes in " + system->systemname + ".csv line: " + line); + lower(root.data()); // alt_route_names - size_t l = 0; - for (size_t r = 0; r != -1; l = r+1) - { r = arn_str.find(',', l); - alt_route_names.emplace_back(arn_str, l, r-l); + upper(arn_str.data()); + size_t len; + for (size_t pos = 0; pos < arn_str.size(); pos += len+1) + { len = strcspn(arn_str.data()+pos, ","); + alt_route_names.emplace_back(arn_str, pos, len); } + + // insert into root_hash, checking for duplicate root entries + if (!root_hash.insert(std::pair(root, this)).second) + el.add_error("Duplicate root in " + system->systemname + ".csv: " + root + + " already in " + root_hash.at(root)->system->systemname + ".csv"); + // insert into list_hash with list name, checking for duplicate .list names + if (!list_hash.insert(std::pair(upper(readable_name()), this)).second) + el.add_error("Duplicate main list name in " + root + ": '" + readable_name() + + "' already points to " + list_hash.at( upper(readable_name()) )->root); + // insert into list_hash with alt names, checking for duplicate .list names + for (std::string& a : alt_route_names) + if (!list_hash.insert(std::pair(upper(rg_str + " " + a), this)).second) + el.add_error("Duplicate alt route name in " + root + ": '" + region->code + ' ' + a + + "' already points to " + list_hash.at(upper(rg_str + " " + a))->root); } std::string Route::str() @@ -179,9 +196,3 @@ void Route::write_nmp_merged(std::string filename) } wptfile.close(); } - -Route *route_by_root(std::string root, std::list &route_list) -{ for (std::list::iterator r = route_list.begin(); r != route_list.end(); r++) - if (r->root == root) return &*r; - return 0; -} diff --git a/siteupdate/cplusplus/classes/Route/Route.h b/siteupdate/cplusplus/classes/Route/Route.h index c67a1479..7da12415 100644 --- a/siteupdate/cplusplus/classes/Route/Route.h +++ b/siteupdate/cplusplus/classes/Route/Route.h @@ -51,6 +51,7 @@ class Route std::vector point_list; std::unordered_set labels_in_use; std::unordered_set unused_alt_labels; + static std::unordered_map root_hash, list_hash; static std::mutex awf_mtx; // for locking the all_wpt_files set when erasing processed WPTs std::mutex liu_mtx; // for locking the labels_in_use set when inserting labels during TravelerList processing std::mutex ual_mtx; // for locking the unused_alt_labels set when removing in-use alt_labels diff --git a/siteupdate/cplusplus/classes/Route/read_wpt.cpp b/siteupdate/cplusplus/classes/Route/read_wpt.cpp index 433c025f..ac7817d3 100644 --- a/siteupdate/cplusplus/classes/Route/read_wpt.cpp +++ b/siteupdate/cplusplus/classes/Route/read_wpt.cpp @@ -63,16 +63,6 @@ void Route::read_wpt continue; } point_list.push_back(w); - // populate unused alt labels - for (size_t i = 0; i < w->alt_labels.size(); i++) - { std::string al = w->alt_labels[i]; - // strip out leading '+' - while (al[0] == '+') al.erase(0, 1); //TODO would erase via iterator be any faster? - // convert to upper case - for (size_t c = 0; c < al.size(); c++) - if (al[c] >= 'a' && al[c] <= 'z') al[c] -= 32; - unused_alt_labels.insert(al); - } all_waypoints->insert(w, 1); // single-point Datachecks, and HighwaySegment diff --git a/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp b/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp index e6419a35..72632136 100644 --- a/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp +++ b/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp @@ -29,7 +29,7 @@ class TravelerList unsigned int preview_systems_clinched; static std::mutex alltrav_mtx; // for locking the traveler_lists list when reading .lists from disk - TravelerList(std::string travname, std::unordered_map *route_hash, ErrorList *el, Arguments *args, std::mutex *strtok_mtx) + TravelerList(std::string travname, ErrorList *el, Arguments *args, std::mutex *strtok_mtx) { active_systems_traveled = 0; active_systems_clinched = 0; preview_systems_traveled = 0; @@ -116,11 +116,11 @@ class TravelerList } // find the root that matches in some system and when we do, match labels - std::string route_entry = lower(std::string(fields[1])); - std::string lookup = std::string(lower(fields[0])) + " " + route_entry; - try { Route *r = route_hash->at(lookup); - for (std:: string a : r->alt_route_names) - if (route_entry == lower(a)) + std::string route_entry = upper(std::string(fields[1])); // leave fields[1] intact for potential AltRouteName note + std::string lookup = std::string(upper(fields[0])) + " " + route_entry; + try { Route *r = Route::list_hash.at(lookup); + for (std::string& a : r->alt_route_names) + if (route_entry == a) { log << "Note: deprecated route name " << fields[1] << " -> canonical name " << r->list_entry_name() << " in line " << trim_line << '\n'; break; @@ -135,28 +135,28 @@ class TravelerList // "+" or "*" when matching std::vector point_indices; unsigned int checking_index = 0; + while (*fields[2] == '*' || *fields[2] == '+') fields[2]++; + while (*fields[3] == '*' || *fields[3] == '+') fields[3]++; + upper(fields[2]); + upper(fields[3]); for (Waypoint *w : r->point_list) - { std::string lower_label = lower(w->label); - while (*lower(fields[2]) == '*' || *fields[2] == '+') fields[2]++; - while (*lower(fields[3]) == '*' || *fields[3] == '+') fields[3]++; - while (lower_label.front() == '+' || lower_label.front() == '*') lower_label.erase(lower_label.begin()); - if (fields[2] == lower_label || fields[3] == lower_label) + { std::string upper_label = upper(w->label); + while (upper_label.front() == '+' || upper_label.front() == '*') upper_label = upper_label.substr(1); + if (fields[2] == upper_label || fields[3] == upper_label) { point_indices.push_back(checking_index); r->liu_mtx.lock(); - r->labels_in_use.insert(upper(lower_label)); + r->labels_in_use.insert(upper_label); r->liu_mtx.unlock(); } else { for (std::string &alt : w->alt_labels) - { lower_label = lower(alt); - while (lower_label.front() == '+') lower_label.erase(lower_label.begin()); - if (fields[2] == lower_label || fields[3] == lower_label) + { if (fields[2] == alt || fields[3] == alt) { point_indices.push_back(checking_index); r->liu_mtx.lock(); - r->labels_in_use.insert(upper(lower_label)); + r->labels_in_use.insert(alt); r->liu_mtx.unlock(); // if we have not yet used this alt label, remove it from the unused set r->ual_mtx.lock(); - r->unused_alt_labels.erase(upper(lower_label)); + r->unused_alt_labels.erase(alt); r->ual_mtx.unlock(); } } diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp index 7b3bfe20..1b84ac51 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp @@ -13,8 +13,6 @@ bool waypoint_simplification_sort(Waypoint *w1, Waypoint *w2) else return 0; } -const double Waypoint::pi = 3.141592653589793238; - Waypoint::Waypoint(char *line, Route *rte, DatacheckEntryList *datacheckerrors) { /* initialize object from a .wpt file line */ route = rte; @@ -314,17 +312,14 @@ bool Waypoint::label_references_route(Route *r, DatacheckEntryList *datacheckerr inline void Waypoint::duplicate_label(DatacheckEntryList *datacheckerrors, std::unordered_set &all_route_labels) { // duplicate labels // first, check primary label - std::string lower_label = lower(label); - while (lower_label[0] == '+' || lower_label[0] == '*') lower_label.erase(lower_label.begin()); - if (!all_route_labels.insert(lower_label).second) - datacheckerrors->add(route, lower_label, "", "", "DUPLICATE_LABEL", ""); + std::string upper_label = upper(label); + while (upper_label[0] == '+' || upper_label[0] == '*') upper_label = upper_label.substr(1); + if (!all_route_labels.insert(upper_label).second) + datacheckerrors->add(route, upper_label, "", "", "DUPLICATE_LABEL", ""); // then check alt labels - for (std::string &label : alt_labels) - { std::string lower_label = lower(label); - while (lower_label[0] == '+' || lower_label[0] == '*') lower_label.erase(lower_label.begin()); - if (!all_route_labels.insert(lower_label).second) - datacheckerrors->add(route, lower_label, "", "", "DUPLICATE_LABEL", ""); - } + for (std::string &a : alt_labels) + if (!all_route_labels.insert(a).second) + datacheckerrors->add(route, a, "", "", "DUPLICATE_LABEL", ""); } inline void Waypoint::duplicate_coords(DatacheckEntryList *datacheckerrors, std::unordered_set &coords_used, char *fstr) diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.h b/siteupdate/cplusplus/classes/Waypoint/Waypoint.h index a419e8d3..e361800a 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.h +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.h @@ -20,7 +20,6 @@ class Waypoint std::vector ap_coloc; std::forward_list near_miss_points; bool is_hidden; - static const double pi; Waypoint(char *, Route *, DatacheckEntryList *); diff --git a/siteupdate/cplusplus/functions/lower.cpp b/siteupdate/cplusplus/functions/lower.cpp index 228d0cde..1947d2c2 100644 --- a/siteupdate/cplusplus/functions/lower.cpp +++ b/siteupdate/cplusplus/functions/lower.cpp @@ -4,8 +4,8 @@ std::string lower(std::string str) return str; } -char *lower(char *str) -{ for (char *c = str; *c != 0; c++) +const char *lower(const char *str) +{ for (char* c = (char*)str; *c != 0; c++) if (*c >= 'A' && *c <= 'Z') *c += 32; return str; } diff --git a/siteupdate/cplusplus/functions/upper.cpp b/siteupdate/cplusplus/functions/upper.cpp index 31a5ae9d..883a1d07 100644 --- a/siteupdate/cplusplus/functions/upper.cpp +++ b/siteupdate/cplusplus/functions/upper.cpp @@ -3,3 +3,9 @@ std::string upper(std::string str) if (str[c] >= 'a' && str[c] <= 'z') str[c] -= 32; return str; } + +const char *upper(const char *str) +{ for (char* c = (char*)str; *c != 0; c++) + if (*c >= 'a' && *c <= 'z') *c -= 32; + return str; +} diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index cef22302..2cc92f38 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -25,6 +25,7 @@ class DatacheckEntryList; class HighwayGraph; class HGVertex; class HGEdge; +#define pi 3.141592653589793238 #include #include #include @@ -255,65 +256,6 @@ int main(int argc, char *argv[]) DatacheckEntryList *datacheckerrors = new DatacheckEntryList; // deleted on termination of program - // check for duplicate .list names - // and duplicate root entries among Route and ConnectedRoute - // data in all highway systems - { cout << et.et() << "Checking for duplicate list names in routes, roots in routes and connected routes." << endl; - unordered_set roots, list_names, duplicate_list_names; - unordered_set con_roots; - - for (HighwaySystem *h : highway_systems) - { for (Route& r : h->route_list) - { if (roots.find(r.root) == roots.end()) roots.insert(r.root); - else el.add_error("Duplicate root in route lists: " + r.root); - string list_name = r.readable_name(); - // FIXME branch based on result of list_names.insert - if (list_names.find(list_name) == list_names.end()) list_names.insert(list_name); - else duplicate_list_names.insert(list_name); - } - for (ConnectedRoute& cr : h->con_route_list) - for (size_t r = 0; r < cr.roots.size(); r++) - // FIXME branch based on result of list_names.insert - if (con_roots.find(cr.roots[r]) == con_roots.end()) con_roots.insert(cr.roots[r]); - else el.add_error("Duplicate root in con_route lists: " + cr.roots[r]->root);//*/ - } - - // Make sure every route was listed as a part of some connected route - if (roots.size() == con_roots.size()) - cout << "Check passed: same number of routes as connected route roots. " << roots.size() << endl; - else { el.add_error("Check FAILED: " + to_string(roots.size()) + " routes != " + to_string(con_roots.size()) + " connected route roots."); - // remove con_routes entries from roots - for (Route *cr : con_roots) - { unordered_set::iterator r = roots.find(cr->root); - if (r != roots.end()) roots.erase(r); //FIXME erase by value - } - // there will be some leftovers, let's look up their routes to make - // an error report entry (not worried about efficiency as there would - // only be a few in reasonable cases) - unsigned int num_found = 0; - for (HighwaySystem *h : highway_systems) - for (Route& r : h->route_list) - for (const string& lr : roots) - if (lr == r.root) - { el.add_error("route " + lr + " not matched by any connected route root."); - num_found++; - break; - } - cout << "Added " << num_found << " ROUTE_NOT_IN_CONNECTED error entries." << endl; - } - - // report any duplicate list names as errors - if (duplicate_list_names.empty()) cout << "No duplicate list names found." << endl; - else { for (string d : duplicate_list_names) el.add_error("Duplicate list name: " + d); - cout << "Added " << duplicate_list_names.size() << " DUPLICATE_LIST_NAME error entries." << endl; - } - - roots.clear(); - list_names.clear(); - duplicate_list_names.clear(); - con_roots.clear(); - } - // For tracking whether any .wpt files are in the directory tree // that do not have a .csv file entry that causes them to be // read into the data @@ -443,13 +385,17 @@ int main(int argc, char *argv[]) #include "functions/concurrency_detection.cpp" - // Create hash table for faster lookup of routes by list file name - cout << et.et() << "Creating route hash table for list processing:" << endl; - unordered_map route_hash; - for (HighwaySystem *h : highway_systems) - for (list::iterator r = h->route_list.begin(); r != h->route_list.end(); r++) //FIXME use more compact syntax (&) - { route_hash[lower(r->readable_name())] = &*r; - for (string &a : r->alt_route_names) route_hash[lower(r->rg_str + " " + a)] = &*r; + cout << et.et() << "Checking for unconnected chopped routes, creating canonical AltLabels & populating unused sets." << endl; + for (HighwaySystem* h : highway_systems) + for (Route& r : h->route_list) + { if (!r.con_route) + el.add_error(r.system->systemname + ".csv: root " + r.root + " not matched by any connected route root."); + for (Waypoint* w : r.point_list) + for (std::string& a : w->alt_labels) + { while (a[0] == '+' || a[0] == '*') a = a.substr(1); + upper(a.data()); + r.unused_alt_labels.insert(a); + } } // Create a list of TravelerList objects, one per person @@ -460,7 +406,7 @@ int main(int argc, char *argv[]) #ifdef threading_enabled // set up for threaded .list file processing for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(ReadListThread, t, &traveler_ids, &id_it, &traveler_lists, &list_mtx, &strtok_mtx, &args, &route_hash, &el); + thr[t] = new thread(ReadListThread, t, &traveler_ids, &id_it, &traveler_lists, &list_mtx, &strtok_mtx, &args, &el); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); for (unsigned int t = 0; t < args.numthreads; t++) @@ -468,7 +414,7 @@ int main(int argc, char *argv[]) #else for (string &t : traveler_ids) { cout << t << ' ' << std::flush; - traveler_lists.push_back(new TravelerList(t, &route_hash, &el, &args, &strtok_mtx)); + traveler_lists.push_back(new TravelerList(t, &el, &args, &strtok_mtx)); } traveler_ids.clear(); #endif diff --git a/siteupdate/cplusplus/threads/ReadListThread.cpp b/siteupdate/cplusplus/threads/ReadListThread.cpp index 5adaa85c..923fb45b 100644 --- a/siteupdate/cplusplus/threads/ReadListThread.cpp +++ b/siteupdate/cplusplus/threads/ReadListThread.cpp @@ -1,7 +1,7 @@ void ReadListThread ( unsigned int id, std::list *traveler_ids, std::list::iterator *it, std::list *traveler_lists, std::mutex *tl_mtx, std::mutex *strtok_mtx, - Arguments *args, std::unordered_map *route_hash, ErrorList *el + Arguments *args, ErrorList *el ) { //printf("Starting ReadListThread %02i\n", id); fflush(stdout); while (*it != traveler_ids->end()) @@ -16,7 +16,7 @@ void ReadListThread //printf("ReadListThread %02i (*it)++\n", id); fflush(stdout); std::cout << tl << ' ' << std::flush; tl_mtx->unlock(); - TravelerList *t = new TravelerList(tl, route_hash, el, args, strtok_mtx); + TravelerList *t = new TravelerList(tl, el, args, strtok_mtx); // deleted on termination of program TravelerList::alltrav_mtx.lock(); traveler_lists->push_back(t); diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 9e764491..873ada68 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -894,6 +894,9 @@ class Route: AltRouteNames: (optional) comma-separated list former or other alternate route names that might appear in user list files. """ + root_hash = dict() + list_hash = dict() + def __init__(self,line,system,el): """initialize object from a .csv file line, but do not yet read in waypoint file""" @@ -936,16 +939,43 @@ def __init__(self,line,system,el): el.add_error("City > " + str(DBFieldLength.city) + " bytes in " + system.systemname + ".csv line: " + line) - self.root = fields[6] + self.root = fields[6].lower() if len(self.root.encode('utf-8')) > DBFieldLength.root: el.add_error("Root > " + str(DBFieldLength.root) + " bytes in " + system.systemname + ".csv line: " + line) - self.alt_route_names = fields[7].split(",") + if len(fields[7]) == 0: + self.alt_route_names = [] + else: + self.alt_route_names = fields[7].upper().split(",") + + # insert into root_hash, checking for duplicate root entries + if self.root in Route.root_hash: + el.add_error("Duplicate root in " + system.systemname + ".csv: " + self.root + + " already in " + Route.root_hash[self.root].system.systemname + ".csv") + else: + Route.root_hash[self.root] = self + # insert into list_hash with list name, checking for duplicate .list names + list_name = self.readable_name().upper() + if list_name in Route.list_hash: + el.add_error("Duplicate main list name in " + self.root + ": '" + self.readable_name() + + "' already points to " + Route.list_hash[list_name].root) + else: + Route.list_hash[list_name] = self + # insert into list_hash with alt names, checking for duplicate .list names + for a in self.alt_route_names: + list_name = self.region.upper() + ' ' + a + if list_name in Route.list_hash: + el.add_error("Duplicate alt route name in " + self.root + ": '" + self.region + ' ' + a + + "' already points to " + Route.list_hash[list_name].root) + else: + Route.list_hash[list_name] = self + self.point_list = [] self.labels_in_use = set() self.unused_alt_labels = set() self.segment_list = [] + self.con_route = None self.mileage = 0.0 self.rootOrder = -1 # order within connected route @@ -976,9 +1006,6 @@ def read_wpt(self,all_waypoints,all_waypoints_lock,datacheckerrors,el,path="../. w = previous_point continue self.point_list.append(w) - # populate unused alt labels - for label in w.alt_labels: - self.unused_alt_labels.add(label.upper().strip("+")) all_waypoints_lock.acquire() # look for near-miss points (before we add this one in) @@ -1086,22 +1113,24 @@ def __init__(self,line,system,el): "_con.csv line: " + line) # fields[4] is the list of roots, which will become a python list # of Route objects already in the system - roots = fields[4].split(",") rootOrder = 0 - for root in roots: - route = None - for check_route in system.route_list: - if check_route.root == root: - route = check_route - break - if route is None: - el.add_error("Could not find Route matching ConnectedRoute root " + root + - " in system " + system.systemname + '.') - else: + for root in fields[4].lower().split(","): + try: + route = Route.root_hash[root] self.roots.append(route) + if route.con_route is not None: + el.add_error("Duplicate root in " + system.systemname + "_con.csv: " + route.root + + " already in " + route.con_route.system.systemname + "_con.csv") + if system != route.system: + el.add_error("System mismatch: chopped route " + route.root + " from " + route.system.systemname + + ".csv in connected route in " + system.systemname + "_con.csv"); + route.con_route = self # save order of route in connected route route.rootOrder = rootOrder - rootOrder += 1 + rootOrder += 1 + except KeyError: + el.add_error("Could not find Route matching ConnectedRoute root " + root + + " in system " + system.systemname + '.') if len(self.roots) < 1: el.add_error("No roots in " + system.systemname + "_con.csv line: " + line) # will be computed for routes in active & preview systems later @@ -1214,7 +1243,7 @@ class TravelerList: start_waypoint end_waypoint """ - def __init__(self,travelername,route_hash,el,path="../../../UserData/list_files"): + def __init__(self,travelername,el,path="../../../UserData/list_files"): list_entries = 0 self.clinched_segments = set() self.traveler_name = travelername[:-5] @@ -1240,17 +1269,17 @@ def __init__(self,travelername,route_hash,el,path="../../../UserData/list_files" continue # find the root that matches in some system and when we do, match labels - route_entry = fields[1].lower() - lookup = fields[0].lower() + ' ' + route_entry - if lookup not in route_hash: + route_entry = fields[1].upper() + lookup = fields[0].upper() + ' ' + route_entry + if lookup not in Route.list_hash: (line, invchar) = no_control_chars(line) self.log_entries.append("Unknown region/highway combo in line: " + line) if invchar: self.log_entries[-1] += " [contains invalid character(s)]" else: - r = route_hash[lookup] + r = Route.list_hash[lookup] for a in r.alt_route_names: - if route_entry == a.lower(): + if route_entry == a: self.log_entries.append("Note: deprecated route name " + fields[1] + " -> canonical name " + r.list_entry_name() + " in line " + line) break @@ -1262,22 +1291,19 @@ def __init__(self,travelername,route_hash,el,path="../../../UserData/list_files" # "+" or "*" when matching point_indices = [] checking_index = 0; + list_label_1 = fields[2].lstrip("+*").upper() + list_label_2 = fields[3].lstrip("+*").upper() for w in r.point_list: - lower_label = w.label.lower().lstrip("+*") - list_label_1 = fields[2].lower().lstrip("+*") - list_label_2 = fields[3].lower().lstrip("+*") - if list_label_1 == lower_label or list_label_2 == lower_label: + upper_label = w.label.lstrip("+*").upper() + if list_label_1 == upper_label or list_label_2 == upper_label: point_indices.append(checking_index) - r.labels_in_use.add(lower_label.upper()) + r.labels_in_use.add(upper_label) else: for alt in w.alt_labels: - lower_label = alt.lower().strip("+") - if list_label_1 == lower_label or list_label_2 == lower_label: + if list_label_1 == alt or list_label_2 == alt: point_indices.append(checking_index) - r.labels_in_use.add(lower_label.upper()) - # if we have not yet used this alt label, remove it from the unused list - if lower_label.upper() in r.unused_alt_labels: - r.unused_alt_labels.remove(lower_label.upper()) + r.labels_in_use.add(alt) + r.unused_alt_labels.discard(alt) checking_index += 1 if len(point_indices) != 2: @@ -2426,69 +2452,6 @@ def __init__(self,filename,descr,vertices,edges,travelers,format,category): # list for datacheck errors that we will need later datacheckerrors = [] -# check for duplicate root entries among Route and ConnectedRoute -# data in all highway systems -print(et.et() + "Checking for duplicate list names in routes, roots in routes and connected routes.",flush=True) -roots = set() -list_names = set() -duplicate_list_names = set() -for h in highway_systems: - for r in h.route_list: - if r.root in roots: - el.add_error("Duplicate root in route lists: " + r.root) - else: - roots.add(r.root) - list_name = r.region + ' ' + r.list_entry_name() - if list_name in list_names: - duplicate_list_names.add(list_name) - else: - list_names.add(list_name) - -con_roots = set() -for h in highway_systems: - for cr in h.con_route_list: - for r in cr.roots: - if r.root in con_roots: - el.add_error("Duplicate root in con_route lists: " + r.root) - else: - con_roots.add(r.root) - -# Make sure every route was listed as a part of some connected route -if len(roots) == len(con_roots): - print("Check passed: same number of routes as connected route roots. " + str(len(roots))) -else: - el.add_error("Check FAILED: " + str(len(roots)) + " routes != " + str(len(con_roots)) + " connected route roots.") - roots = roots - con_roots - # there will be some leftovers, let's look up their routes to make - # an error report entry (not worried about efficiency as there would - # only be a few in reasonable cases) - num_found = 0 - for h in highway_systems: - for r in h.route_list: - for lr in roots: - if lr == r.root: - el.add_error("route " + lr + " not matched by any connected route root.") - num_found += 1 - break - print("Added " + str(num_found) + " ROUTE_NOT_IN_CONNECTED error entries.") - -# report any duplicate list names as errors -if len(duplicate_list_names) > 0: - print("Found " + str(len(duplicate_list_names)) + " DUPLICATE_LIST_NAME case(s).") - for d in duplicate_list_names: - el.add_error("Duplicate list name: " + d) -else: - print("No duplicate list names found.") - -# write file mapping CHM datacheck route lists to root (commented out, -# unlikely needed now) -#print(et.et() + "Writing CHM datacheck to TravelMapping route pairings.") -#file = open(args.csvstatfilepath + "/routepairings.csv","wt") -#for h in highway_systems: -# for r in h.route_list: -# file.write(r.region + " " + r.list_entry_name() + ";" + r.root + "\n") -#file.close() - # For tracking whether any .wpt files are in the directory tree # that do not have a .csv file entry that causes them to be # read into the data @@ -2718,14 +2681,15 @@ def run(self): print(".", end="", flush=True) print() -# Create hash table for faster lookup of routes by list file name -print(et.et() + "Creating route hash table for list processing:",flush=True) -route_hash = dict() +print(et.et() + "Checking for unconnected chopped routes, creating canonical AltLabels & populating unused sets.",flush=True) for h in highway_systems: for r in h.route_list: - route_hash[(r.region + ' ' + r.list_entry_name()).lower()] = r - for a in r.alt_route_names: - route_hash[(r.region + ' ' + a).lower()] = r + if r.con_route is None: + el.add_error(r.system.systemname + ".csv: root " + r.root + " not matched by any connected route root.") + for w in r.point_list: + for a in range(len(w.alt_labels)): + w.alt_labels[a] = w.alt_labels[a].lstrip('+*').upper() + r.unused_alt_labels.add(w.alt_labels[a]) # Create a list of TravelerList objects, one per person traveler_lists = [] @@ -2734,7 +2698,7 @@ def run(self): for t in traveler_ids: if t.endswith('.list'): print(t + " ",end="",flush=True) - traveler_lists.append(TravelerList(t,route_hash,el,args.userlistfilepath)) + traveler_lists.append(TravelerList(t,el,args.userlistfilepath)) print('\n' + et.et() + "Processed " + str(len(traveler_lists)) + " traveler list files.") traveler_lists.sort(key=lambda TravelerList: TravelerList.traveler_name) # assign traveler numbers @@ -3605,18 +3569,17 @@ def run(self): for w in r.point_list: # duplicate labels # first, check primary label - lower_label = w.label.lower().strip("+*") - if lower_label in all_route_labels: - datacheckerrors.append(DatacheckEntry(r,[lower_label],"DUPLICATE_LABEL")) + upper_label = w.label.strip("+*").upper() + if upper_label in all_route_labels: + datacheckerrors.append(DatacheckEntry(r,[upper_label],"DUPLICATE_LABEL")) else: - all_route_labels.add(lower_label) + all_route_labels.add(upper_label) # then check alt labels - for label in w.alt_labels: - lower_label = label.lower().strip("+*") - if lower_label in all_route_labels: - datacheckerrors.append(DatacheckEntry(r,[lower_label],"DUPLICATE_LABEL")) + for a in w.alt_labels: + if a in all_route_labels: + datacheckerrors.append(DatacheckEntry(r,[a],"DUPLICATE_LABEL")) else: - all_route_labels.add(lower_label) + all_route_labels.add(a) # out-of-bounds coords if w.lat > 90 or w.lat < -90 or w.lng > 180 or w.lng < -180: From f50764d0ee137d66ff32a543c00ff2260d6bf0c0 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Wed, 13 May 2020 23:03:35 -0400 Subject: [PATCH 07/34] range for loop cleanup * Retrieve elements by value rather than reference. Same principle as in #270, just without the big speed increase. Best practice nonetheless. * Convert traditional for loops to range for loops where possible, for cleaner more concise code. --- .../classes/GraphGeneration/HighwayGraph.cpp | 4 +- .../classes/HighwaySegment/HighwaySegment.cpp | 16 +++--- .../cplusplus/classes/HighwaySystem.cpp | 2 +- siteupdate/cplusplus/classes/Region.cpp | 4 +- .../classes/TravelerList/TravelerList.cpp | 18 +++---- .../WaypointQuadtree/WaypointQuadtree.cpp | 4 +- .../cplusplus/functions/graph_generation.cpp | 5 +- siteupdate/cplusplus/functions/sql_file.cpp | 4 +- siteupdate/cplusplus/siteupdate.cpp | 51 +++++++++---------- 9 files changed, 52 insertions(+), 56 deletions(-) diff --git a/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp b/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp index 8ae6bfaf..238d9d6b 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp +++ b/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp @@ -96,7 +96,7 @@ class HighwayGraph // compress edges adjacent to hidden vertices counter = 0; std::cout << et.et() + "Compressing collapsed edges" << std::flush; - for (std::pair wv : vertices) + for (std::pair& wv : vertices) { if (counter % 10000 == 0) std::cout << '.' << std::flush; counter++; if (!wv.second->visibility) @@ -146,7 +146,7 @@ class HighwayGraph } // end ctor void clear() - { for (std::pair wv : vertices) delete wv.second; + { for (std::pair& wv : vertices) delete wv.second; vertex_names.clear(); waypoint_naming_log.clear(); vertices.clear(); diff --git a/siteupdate/cplusplus/classes/HighwaySegment/HighwaySegment.cpp b/siteupdate/cplusplus/classes/HighwaySegment/HighwaySegment.cpp index 30cccdf1..4b5d009f 100644 --- a/siteupdate/cplusplus/classes/HighwaySegment/HighwaySegment.cpp +++ b/siteupdate/cplusplus/classes/HighwaySegment/HighwaySegment.cpp @@ -49,19 +49,15 @@ unsigned int HighwaySegment::index() /*std::string HighwaySegment::concurrent_travelers_sanity_check() { if (route->system->devel()) return ""; if (concurrent) - for (HighwaySegment *conc : *concurrent) - { if (clinched_by.size() != conc->clinched_by.size()) - { if (conc->route->system->devel()) continue; + for (HighwaySegment *other : *concurrent) + { if (clinched_by.size() != other->clinched_by.size()) + { if (other->route->system->devel()) continue; return "[" + str() + "] clinched by " + std::to_string(clinched_by.size()) + " travelers; [" \ - + conc->str() + "] clinched by " + std::to_string(conc->clinched_by.size()) + '\n'; + + other->str() + "] clinched by " + std::to_string(other->clinched_by.size()) + '\n'; } else for (TravelerList *t : clinched_by) - { std::list::iterator ct; - for (ct = conc->clinched_by.begin(); ct != conc->clinched_by.end(); ct++) - if (*ct == t) break; - if (ct == conc->clinched_by.end()) - return t->traveler_name + " has clinched [" + str() + "], but not [" + conc->str() + "]\n"; - } + if (other->clinched_by.find(t) == other->clinched_by.end()) + return t->traveler_name + " has clinched [" + str() + "], but not [" + other->str() + "]\n"; } return ""; }//*/ diff --git a/siteupdate/cplusplus/classes/HighwaySystem.cpp b/siteupdate/cplusplus/classes/HighwaySystem.cpp index 0d6bc2b3..2ee68d96 100644 --- a/siteupdate/cplusplus/classes/HighwaySystem.cpp +++ b/siteupdate/cplusplus/classes/HighwaySystem.cpp @@ -128,7 +128,7 @@ class HighwaySystem /* Return total system mileage across all regions */ double total_mileage() { double mi = 0; - for (std::pair rm : mileage_by_region) mi += rm.second; + for (std::pair& rm : mileage_by_region) mi += rm.second; return mi; } diff --git a/siteupdate/cplusplus/classes/Region.cpp b/siteupdate/cplusplus/classes/Region.cpp index 4716aa2e..d7daa493 100644 --- a/siteupdate/cplusplus/classes/Region.cpp +++ b/siteupdate/cplusplus/classes/Region.cpp @@ -1,6 +1,6 @@ std::pair *country_or_continent_by_code(std::string code, std::vector> &pair_vector) -{ for (std::vector>::iterator c = pair_vector.begin(); c != pair_vector.end(); c++) - if (c->first == code) return &*c; +{ for (std::pair& c : pair_vector) + if (c.first == code) return &c; return 0; } diff --git a/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp b/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp index 72632136..d2a9ded1 100644 --- a/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp +++ b/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp @@ -165,9 +165,9 @@ class TravelerList } if (point_indices.size() != 2) { bool invalid_char = 0; - for (size_t c = 0; c < trim_line.size(); c++) - if (iscntrl(trim_line[c])) - { trim_line[c] = '?'; + for (char& c : trim_line) + if (iscntrl(c)) + { c = '?'; invalid_char = 1; } log << "Waypoint label(s) not found in line: " << trim_line; @@ -189,9 +189,9 @@ class TravelerList } catch (const std::out_of_range& oor) { bool invalid_char = 0; - for (size_t c = 0; c < trim_line.size(); c++) - if (iscntrl(trim_line[c])) - { trim_line[c] = '?'; + for (char& c : trim_line) + if (iscntrl(c)) + { c = '?'; invalid_char = 1; } log << "Unknown region/highway combo in line: " << trim_line; @@ -209,21 +209,21 @@ class TravelerList /* Return active mileage across all regions */ double active_only_miles() { double mi = 0; - for (std::pair rm : active_only_mileage_by_region) mi += rm.second; + for (std::pair& rm : active_only_mileage_by_region) mi += rm.second; return mi; } /* Return active+preview mileage across all regions */ double active_preview_miles() { double mi = 0; - for (std::pair rm : active_preview_mileage_by_region) mi += rm.second; + for (std::pair& rm : active_preview_mileage_by_region) mi += rm.second; return mi; } /* Return mileage across all regions for a specified system */ double system_region_miles(HighwaySystem *h) { double mi = 0; - for (std::pair rm : system_region_mileages.at(h)) mi += rm.second; + for (std::pair& rm : system_region_mileages.at(h)) mi += rm.second; return mi; } diff --git a/siteupdate/cplusplus/classes/WaypointQuadtree/WaypointQuadtree.cpp b/siteupdate/cplusplus/classes/WaypointQuadtree/WaypointQuadtree.cpp index b6285089..e6e86243 100644 --- a/siteupdate/cplusplus/classes/WaypointQuadtree/WaypointQuadtree.cpp +++ b/siteupdate/cplusplus/classes/WaypointQuadtree/WaypointQuadtree.cpp @@ -259,7 +259,7 @@ void WaypointQuadtree::write_qt_tmg(std::string filename) std::ofstream tmgfile(filename); tmgfile << "TMG 1.0 simple\n"; tmgfile << vertices.size() << ' ' << edges.size() << '\n'; - for (std::string v : vertices) tmgfile << v << '\n'; - for (std::string e : edges) tmgfile << e << '\n'; + for (std::string& v : vertices) tmgfile << v << '\n'; + for (std::string& e : edges) tmgfile << e << '\n'; tmgfile.close(); } diff --git a/siteupdate/cplusplus/functions/graph_generation.cpp b/siteupdate/cplusplus/functions/graph_generation.cpp index 02b3e0d6..9b47f0f3 100644 --- a/siteupdate/cplusplus/functions/graph_generation.cpp +++ b/siteupdate/cplusplus/functions/graph_generation.cpp @@ -5,7 +5,7 @@ HighwayGraph graph_data(all_waypoints, highway_systems, datacheckerrors, args.nu cout << et.et() << "Writing graph waypoint simplification log." << endl; ofstream wslogfile(args.logfilepath + "/waypointsimplification.log"); -for (string line : graph_data.waypoint_naming_log) +for (string& line : graph_data.waypoint_naming_log) wslogfile << line << '\n'; wslogfile.close(); graph_data.waypoint_naming_log.clear(); @@ -34,7 +34,8 @@ else { list *regions; graph_vector[0].edges = 0; graph_vector[0].vertices = graph_data.vertices.size(); graph_vector[1].edges = 0; graph_vector[1].vertices = 0; graph_vector[2].edges = 0; graph_vector[2].vertices = 0; - for (std::pair wv : graph_data.vertices) + // get master graph vertex & edge counts for terminal output before writing files + for (std::pair& wv : graph_data.vertices) { graph_vector[0].edges += wv.second->incident_s_edges.size(); if (wv.second->visibility >= 1) { graph_vector[2].vertices++; diff --git a/siteupdate/cplusplus/functions/sql_file.cpp b/siteupdate/cplusplus/functions/sql_file.cpp index ac9c05d0..2317eefb 100644 --- a/siteupdate/cplusplus/functions/sql_file.cpp +++ b/siteupdate/cplusplus/functions/sql_file.cpp @@ -273,7 +273,7 @@ void sqlfile1 first = 1; for (HighwaySystem *h : *highway_systems) if (h->active_or_preview()) - for (std::pair rm : h->mileage_by_region) + for (std::pair& rm : h->mileage_by_region) { if (!first) sqlfile << ','; first = 0; char fstr[35]; @@ -293,7 +293,7 @@ void sqlfile1 sqlfile << "INSERT INTO clinchedOverallMileageByRegion VALUES\n"; first = 1; for (TravelerList *t : *traveler_lists) - for (std::pair rm : t->active_preview_mileage_by_region) + for (std::pair& rm : t->active_preview_mileage_by_region) { if (!first) sqlfile << ','; first = 0; double active_miles = 0; diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index 2cc92f38..7ab1b1d1 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -248,7 +248,7 @@ int main(int argc, char *argv[]) } cout << endl; // at the end, print the lines ignored - for (string l : ignoring) cout << l << endl; + for (string& l : ignoring) cout << l << endl; ignoring.clear(); } file.close(); @@ -285,9 +285,8 @@ int main(int argc, char *argv[]) #else for (HighwaySystem* h : highway_systems) { std::cout << h->systemname << std::flush; - for (std::list::iterator r = h->route_list.begin(); r != h->route_list.end(); r++) - { r->read_wpt(&all_waypoints, &el, args.highwaydatapath+"/hwy_data", datacheckerrors, &all_wpt_files); - } + for (Route& r : h->route_list) + r.read_wpt(&all_waypoints, &el, args.highwaydatapath+"/hwy_data", datacheckerrors, &all_wpt_files); std::cout << "!" << std::endl; } #endif @@ -529,7 +528,7 @@ int main(int argc, char *argv[]) { inusefile << r.root << '(' << r.point_list.size() << "):"; list liu_list(r.labels_in_use.begin(), r.labels_in_use.end()); liu_list.sort(); - for (string label : liu_list) inusefile << ' ' << label; + for (string& label : liu_list) inusefile << ' ' << label; inusefile << '\n'; r.labels_in_use.clear(); } @@ -546,7 +545,7 @@ int main(int argc, char *argv[]) string ual_entry = r.root + '(' + to_string(r.unused_alt_labels.size()) + "):"; list ual_list(r.unused_alt_labels.begin(), r.unused_alt_labels.end()); ual_list.sort(); - for (string label : ual_list) ual_entry += ' ' + label; + for (string& label : ual_list) ual_entry += ' ' + label; r.unused_alt_labels.clear(); unused_alt_labels.push_back(ual_entry); } @@ -656,7 +655,7 @@ int main(int argc, char *argv[]) region_entries.push_back(region->code + fstr); } region_entries.sort(); - for (string e : region_entries) hdstatsfile << e; + for (string& e : region_entries) hdstatsfile << e; for (HighwaySystem *h : highway_systems) { sprintf(fstr, ") total: %.2f mi\n", h->total_mileage()); @@ -664,7 +663,7 @@ int main(int argc, char *argv[]) if (h->mileage_by_region.size() > 1) { hdstatsfile << "System " << h->systemname << " by region:\n"; list regions_in_system; - for (pair rm : h->mileage_by_region) + for (pair& rm : h->mileage_by_region) regions_in_system.push_back(rm.first); regions_in_system.sort(sort_regions_by_code); for (Region *r : regions_in_system) @@ -673,19 +672,19 @@ int main(int argc, char *argv[]) } } hdstatsfile << "System " << h->systemname << " by route:\n"; - for (list::iterator cr = h->con_route_list.begin(); cr != h->con_route_list.end(); cr++) + for (ConnectedRoute& cr : h->con_route_list) { double con_total_miles = 0; string to_write = ""; - for (Route *r : cr->roots) + for (Route *r : cr.roots) { sprintf(fstr, ": %.2f mi\n", r->mileage); to_write += " " + r->readable_name() + fstr; con_total_miles += r->mileage; } - cr->mileage = con_total_miles; //FIXME? + cr.mileage = con_total_miles; //FIXME? sprintf(fstr, ": %.2f mi", con_total_miles); - hdstatsfile << cr->readable_name() << fstr; - if (cr->roots.size() == 1) - hdstatsfile << " (" << cr->roots[0]->readable_name() << " only)\n"; + hdstatsfile << cr.readable_name() << fstr; + if (cr.roots.size() == 1) + hdstatsfile << " (" << cr.roots[0]->readable_name() << " only)\n"; else hdstatsfile << '\n' << to_write; } } @@ -733,7 +732,7 @@ int main(int argc, char *argv[]) allfile << '\n'; for (TravelerList *t : traveler_lists) { double t_total_mi = 0; - for (std::pair rm : t->active_only_mileage_by_region) + for (std::pair& rm : t->active_only_mileage_by_region) t_total_mi += rm.second; sprintf(fstr, "%.2f", t_total_mi); allfile << t->traveler_name << ',' << fstr; @@ -771,7 +770,7 @@ int main(int argc, char *argv[]) allfile << '\n'; for (TravelerList *t : traveler_lists) { double t_total_mi = 0; - for (std::pair rm : t->active_preview_mileage_by_region) + for (std::pair& rm : t->active_preview_mileage_by_region) t_total_mi += rm.second; sprintf(fstr, "%.2f", t_total_mi); allfile << t->traveler_name << ',' << fstr; @@ -799,7 +798,7 @@ int main(int argc, char *argv[]) sysfile << "Traveler,Total"; regions.clear(); total_mi = 0; - for (std::pair rm : h->mileage_by_region) + for (std::pair& rm : h->mileage_by_region) { regions.push_back(rm.first); total_mi += rm.second; //TODO is this right? } @@ -835,7 +834,7 @@ int main(int argc, char *argv[]) cout << et.et() << "Reading datacheckfps.csv." << endl; file.open(args.highwaydatapath+"/datacheckfps.csv"); getline(file, line); // ignore header line - list> datacheckfps; //FIXME try implementing as an unordered_multiset; see if speed increases + list> datacheckfps; unordered_set datacheck_always_error ({ "BAD_ANGLE", "DUPLICATE_LABEL", "HIDDEN_TERMINUS", "INVALID_FINAL_CHAR", "INVALID_FIRST_CHAR", @@ -881,22 +880,22 @@ int main(int argc, char *argv[]) fpfile << "Log file created at: " << ctime(×tamp); unsigned int counter = 0; unsigned int fpcount = 0; - for (list::iterator d = datacheckerrors->entries.begin(); d != datacheckerrors->entries.end(); d++) + for (DatacheckEntry& d : datacheckerrors->entries) { //cout << "Checking: " << d->str() << endl; counter++; if (counter % 1000 == 0) cout << '.' << flush; for (list>::iterator fp = datacheckfps.begin(); fp != datacheckfps.end(); fp++) - if (d->match_except_info(*fp)) - if (d->info == (*fp)[5]) + if (d.match_except_info(*fp)) + if (d.info == (*fp)[5]) { //cout << "Match!" << endl; - d->fp = 1; + d.fp = 1; fpcount++; datacheckfps.erase(fp); break; } else { fpfile << "FP_ENTRY: " << (*fp)[0] << ';' << (*fp)[1] << ';' << (*fp)[2] << ';' << (*fp)[3] << ';' << (*fp)[4] << ';' << (*fp)[5] << '\n'; - fpfile << "CHANGETO: " << (*fp)[0] << ';' << (*fp)[1] << ';' << (*fp)[2] << ';' << (*fp)[3] << ';' << (*fp)[4] << ';' << d->info << '\n'; + fpfile << "CHANGETO: " << (*fp)[0] << ';' << (*fp)[1] << ';' << (*fp)[2] << ';' << (*fp)[3] << ';' << (*fp)[4] << ';' << d.info << '\n'; } } fpfile.close(); @@ -910,8 +909,8 @@ int main(int argc, char *argv[]) timestamp = time(0); fpfile << "Log file created at: " << ctime(×tamp); if (datacheckfps.empty()) fpfile << "No unmatched FP entries.\n"; - else for (array entry : datacheckfps) - fpfile << entry[0] << ';' << entry[1] << ';' << entry[2] << ';' << entry[3] << ';' << entry[4] << ';' << entry[5] << '\n'; + else for (array& entry : datacheckfps) + fpfile << entry[0] << ';' << entry[1] << ';' << entry[2] << ';' << entry[3] << ';' << entry[4] << ';' << entry[5] << '\n'; fpfile.close(); // datacheck.log file @@ -999,7 +998,7 @@ int main(int argc, char *argv[]) unsigned other_count = 0; unsigned int total_rtes = 0; for (HighwaySystem *h : highway_systems) - for (Route r : h->route_list) + for (Route& r : h->route_list) { total_rtes++; if (h->devel()) d_count++; else { if (h->active()) a_count++; From 11d0ca783d8142c2ba52c490526b7ff7c8bbef54 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Wed, 20 May 2020 18:42:06 -0400 Subject: [PATCH 08/34] .list processing enhancements * C++: bugfix for duplicate AltLabels datacheck * closes #325 * hash tables for waypoint labels * closes #278 * C++: parallelism processing .list files * closes #274 * improved userlog error messages * closes #326 * closes #275 * listnamesinuse.log & unusedaltroutenames.log * closes #230 * preparation for #58 * #308 * [Route::store_traveled_segments](https://github.com/TravelMapping/DataProcessing/issues/58#issuecomment-629957813) * C++: refactor chopped route (4-field) list line parsing into its own file --- .../cplusplus/classes/HighwaySystem.cpp | 3 +- siteupdate/cplusplus/classes/Route/Route.cpp | 42 ++- siteupdate/cplusplus/classes/Route/Route.h | 5 +- .../cplusplus/classes/Route/read_wpt.cpp | 9 +- .../classes/TravelerList/TravelerList.cpp | 326 ++++++------------ .../classes/TravelerList/TravelerList.h | 37 ++ .../mark_chopped_route_segments.cpp | 111 ++++++ .../classes/TravelerList/splitregion.cpp | 4 +- .../classes/TravelerList/userlog.cpp | 2 +- .../cplusplus/classes/Waypoint/Waypoint.cpp | 19 +- .../cplusplus/classes/Waypoint/Waypoint.h | 1 - .../WaypointQuadtree/WaypointQuadtree.cpp | 2 +- .../WaypointQuadtree/WaypointQuadtree.h | 2 +- siteupdate/cplusplus/siteupdate.cpp | 148 +++++--- .../cplusplus/threads/ReadListThread.cpp | 5 +- siteupdate/python-teresco/siteupdate.py | 293 ++++++++++------ 16 files changed, 599 insertions(+), 410 deletions(-) create mode 100644 siteupdate/cplusplus/classes/TravelerList/TravelerList.h create mode 100644 siteupdate/cplusplus/classes/TravelerList/mark_chopped_route_segments.cpp diff --git a/siteupdate/cplusplus/classes/HighwaySystem.cpp b/siteupdate/cplusplus/classes/HighwaySystem.cpp index 2ee68d96..e63f63c1 100644 --- a/siteupdate/cplusplus/classes/HighwaySystem.cpp +++ b/siteupdate/cplusplus/classes/HighwaySystem.cpp @@ -1,4 +1,3 @@ -//FIXME try to break strtok. That goes for all strtok project-wide. class HighwaySystem { /* This class encapsulates the contents of one .csv file that represents the collection of highways within a system. @@ -28,6 +27,8 @@ class HighwaySystem std::list con_route_list; std::unordered_map mileage_by_region; std::unordered_set vertices; + std::unordered_setlistnamesinuse, unusedaltroutenames; + std::mutex lniu_mtx, uarn_mtx; bool is_valid; HighwaySystem(std::string &line, ErrorList &el, std::string path, std::string &systemsfile, diff --git a/siteupdate/cplusplus/classes/Route/Route.cpp b/siteupdate/cplusplus/classes/Route/Route.cpp index 4cbf193c..1638cd3b 100644 --- a/siteupdate/cplusplus/classes/Route/Route.cpp +++ b/siteupdate/cplusplus/classes/Route/Route.cpp @@ -1,4 +1,4 @@ -std::unordered_map Route::root_hash, Route::list_hash; +std::unordered_map Route::root_hash, Route::pri_list_hash, Route::alt_list_hash; std::mutex Route::awf_mtx; Route::Route(std::string &line, HighwaySystem *sys, ErrorList &el, std::unordered_map ®ion_hash) @@ -64,17 +64,28 @@ Route::Route(std::string &line, HighwaySystem *sys, ErrorList &el, std::unordere // insert into root_hash, checking for duplicate root entries if (!root_hash.insert(std::pair(root, this)).second) - el.add_error("Duplicate root in " + system->systemname + ".csv: " + root + - " already in " + root_hash.at(root)->system->systemname + ".csv"); - // insert into list_hash with list name, checking for duplicate .list names - if (!list_hash.insert(std::pair(upper(readable_name()), this)).second) - el.add_error("Duplicate main list name in " + root + ": '" + readable_name() + - "' already points to " + list_hash.at( upper(readable_name()) )->root); - // insert into list_hash with alt names, checking for duplicate .list names + el.add_error("Duplicate root in " + system->systemname + ".csv: " + root + + " already in " + root_hash.at(root)->system->systemname + ".csv"); + // insert list name into pri_list_hash, checking for duplicate .list names + std::string list_name = upper(readable_name()); + if (alt_list_hash.find(list_name) != alt_list_hash.end()) + el.add_error("Duplicate main list name in " + root + ": '" + readable_name() + + "' already points to " + alt_list_hash.at(list_name)->root); + else if (!pri_list_hash.insert(std::pair(list_name, this)).second) + el.add_error("Duplicate main list name in " + root + ": '" + readable_name() + + "' already points to " + pri_list_hash.at(list_name)->root); + // insert alt names into alt_list_hash, checking for duplicate .list names for (std::string& a : alt_route_names) - if (!list_hash.insert(std::pair(upper(rg_str + " " + a), this)).second) - el.add_error("Duplicate alt route name in " + root + ": '" + region->code + ' ' + a + - "' already points to " + list_hash.at(upper(rg_str + " " + a))->root); + { list_name = upper(rg_str + ' ' + a); + if (pri_list_hash.find(list_name) != pri_list_hash.end()) + el.add_error("Duplicate alt route name in " + root + ": '" + region->code + ' ' + a + + "' already points to " + pri_list_hash.at(list_name)->root); + else if (!alt_list_hash.insert(std::pair(list_name, this)).second) + el.add_error("Duplicate alt route name in " + root + ": '" + region->code + ' ' + a + + "' already points to " + alt_list_hash.at(list_name)->root); + // populate unused set + system->unusedaltroutenames.insert(list_name); + } } std::string Route::str() @@ -196,3 +207,12 @@ void Route::write_nmp_merged(std::string filename) } wptfile.close(); } + +inline void Route::store_traveled_segments(TravelerList* t, unsigned int beg, unsigned int end) +{ // store clinched segments with traveler and traveler with segments + for (unsigned int pos = beg; pos < end; pos++) + { HighwaySegment *hs = segment_list[pos]; + hs->add_clinched_by(t); + t->clinched_segments.insert(hs); + } +} diff --git a/siteupdate/cplusplus/classes/Route/Route.h b/siteupdate/cplusplus/classes/Route/Route.h index 7da12415..eaec3919 100644 --- a/siteupdate/cplusplus/classes/Route/Route.h +++ b/siteupdate/cplusplus/classes/Route/Route.h @@ -51,7 +51,9 @@ class Route std::vector point_list; std::unordered_set labels_in_use; std::unordered_set unused_alt_labels; - static std::unordered_map root_hash, list_hash; + std::unordered_set duplicate_labels; + std::unordered_map pri_label_hash, alt_label_hash; + static std::unordered_map root_hash, pri_list_hash, alt_list_hash; static std::mutex awf_mtx; // for locking the all_wpt_files set when erasing processed WPTs std::mutex liu_mtx; // for locking the labels_in_use set when inserting labels during TravelerList processing std::mutex ual_mtx; // for locking the unused_alt_labels set when removing in-use alt_labels @@ -73,4 +75,5 @@ class Route double clinched_by_traveler(TravelerList *); std::string list_line(int, int); void write_nmp_merged(std::string); + inline void store_traveled_segments(TravelerList*, unsigned int, unsigned int); }; diff --git a/siteupdate/cplusplus/classes/Route/read_wpt.cpp b/siteupdate/cplusplus/classes/Route/read_wpt.cpp index ac7817d3..f8d3db14 100644 --- a/siteupdate/cplusplus/classes/Route/read_wpt.cpp +++ b/siteupdate/cplusplus/classes/Route/read_wpt.cpp @@ -27,17 +27,11 @@ void Route::read_wpt // split file into lines size_t spn = 0; for (char* c = wptdata; *c; c += spn) - { spn = strcspn(c, "\n\r"); - while (c[spn] == '\n' || c[spn] == '\r') - { c[spn] = 0; - spn++; - } + { for (spn = strcspn(c, "\n\r"); c[spn] == '\n' || c[spn] == '\r'; spn++) c[spn] = 0; lines.emplace_back(c); } lines.push_back(wptdata+wptdatasize+1); // add a dummy "past-the-end" element to make lines[l+1]-2 work - // set to be used per-route to find label duplicates - std::unordered_set all_route_labels; // set to be used for finding duplicate coordinates std::unordered_set coords_used; double vis_dist = 0; @@ -67,7 +61,6 @@ void Route::read_wpt // single-point Datachecks, and HighwaySegment w->out_of_bounds(datacheckerrors, fstr); - w->duplicate_label(datacheckerrors, all_route_labels); w->duplicate_coords(datacheckerrors, coords_used, fstr); if (point_list.size() > 1) { w->distance_update(datacheckerrors, fstr, vis_dist, point_list[point_list.size()-2]); diff --git a/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp b/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp index d2a9ded1..b92dc698 100644 --- a/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp +++ b/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp @@ -1,234 +1,118 @@ -class TravelerList -{ /* This class encapsulates the contents of one .list file - that represents the travels of one individual user. +TravelerList::TravelerList(std::string travname, ErrorList *el, Arguments *args) +{ active_systems_traveled = 0; + active_systems_clinched = 0; + preview_systems_traveled = 0; + preview_systems_clinched = 0; + unsigned int list_entries = 0; + traveler_num = new unsigned int[args->numthreads]; + // deleted on termination of program + traveler_name = travname.substr(0, travname.size()-5); // strip ".list" from end of travname + if (traveler_name.size() > DBFieldLength::traveler) + el->add_error("Traveler name " + traveler_name + " > " + std::to_string(DBFieldLength::traveler) + "bytes"); + std::ofstream log(args->logfilepath+"/users/"+traveler_name+".log"); + std::ofstream splist; + if (args->splitregionpath != "") splist.open(args->splitregionpath+"/list_files/"+travname); + time_t StartTime = time(0); + log << "Log file created at: " << ctime(&StartTime); + std::vector lines; + std::vector endlines; + std::ifstream file(args->userlistfilepath+"/"+travname); + // we can't getline here because it only allows one delimiter, and we need two; '\r' and '\n'. + // at least one .list file contains newlines using only '\r' (0x0D): + // https://github.com/TravelMapping/UserData/blob/6309036c44102eb3325d49515b32c5eef3b3cb1e/list_files/whopperman.list + file.seekg(0, std::ios::end); + unsigned long listdatasize = file.tellg(); + file.seekg(0, std::ios::beg); + char *listdata = new char[listdatasize+1]; + file.read(listdata, listdatasize); + listdata[listdatasize] = 0; // add null terminator + file.close(); - A list file consists of lines of 4 values: - region route_name start_waypoint end_waypoint + // get canonical newline for writing splitregion .list files + std::string newline; + unsigned long c = 0; + while (listdata[c] != '\r' && listdata[c] != '\n' && c < listdatasize) c++; + if (listdata[c] == '\r') + if (listdata[c+1] == '\n') newline = "\r\n"; + else newline = "\r"; + else if (listdata[c] == '\n') newline = "\n"; + // Use CRLF as failsafe if .list file contains no newlines. + else newline = "\r\n"; - which indicates that the user has traveled the highway names - route_name in the given region between the waypoints named - start_waypoint end_waypoint - */ - public: - std::mutex ap_mi_mtx, ao_mi_mtx, sr_mi_mtx; - std::unordered_set clinched_segments; - std::string traveler_name; - std::unordered_map active_preview_mileage_by_region; // total mileage per region, active+preview only - std::unordered_map active_only_mileage_by_region; // total mileage per region, active only - std::unordered_map> system_region_mileages; // mileage per region per system - std::unordered_map> con_routes_traveled; // mileage per ConRte per system - // TODO is this necessary? - // ConRtes by definition exist in one system only - std::unordered_map routes_traveled; // mileage per traveled route - std::unordered_map con_routes_clinched; // clinch count per system - //std::unordered_map routes_clinched; // commented out in original siteupdate.py - unsigned int *traveler_num; - unsigned int active_systems_traveled; - unsigned int active_systems_clinched; - unsigned int preview_systems_traveled; - unsigned int preview_systems_clinched; - static std::mutex alltrav_mtx; // for locking the traveler_lists list when reading .lists from disk - - TravelerList(std::string travname, ErrorList *el, Arguments *args, std::mutex *strtok_mtx) - { active_systems_traveled = 0; - active_systems_clinched = 0; - preview_systems_traveled = 0; - preview_systems_clinched = 0; - unsigned int list_entries = 0; - traveler_num = new unsigned int[args->numthreads]; - // deleted on termination of program - traveler_name = travname.substr(0, travname.size()-5); // strip ".list" from end of travname - if (traveler_name.size() > DBFieldLength::traveler) - el->add_error("Traveler name " + traveler_name + " > " + std::to_string(DBFieldLength::traveler) + "bytes"); - std::ofstream log(args->logfilepath+"/users/"+traveler_name+".log"); - std::ofstream splist; - if (args->splitregionpath != "") splist.open(args->splitregionpath+"/list_files/"+travname); - time_t StartTime = time(0); - log << "Log file created at: " << ctime(&StartTime); - std::vector lines; - std::vector endlines; - std::ifstream file(args->userlistfilepath+"/"+travname); - // we can't getline here because it only allows one delimiter, and we need two; '\r' and '\n'. - // at least one .list file contains newlines using only '\r' (0x0D): - // https://github.com/TravelMapping/UserData/blob/6309036c44102eb3325d49515b32c5eef3b3cb1e/list_files/whopperman.list - file.seekg(0, std::ios::end); - unsigned long listdatasize = file.tellg(); - file.seekg(0, std::ios::beg); - char *listdata = new char[listdatasize+1]; - file.read(listdata, listdatasize); - listdata[listdatasize] = 0; // add null terminator - file.close(); - - // get canonical newline for writing splitregion .list files - std::string newline; - unsigned long c = 0; - while (listdata[c] != '\r' && listdata[c] != '\n' && c < listdatasize) c++; - if (listdata[c] == '\r') - if (listdata[c+1] == '\n') newline = "\r\n"; - else newline = "\r"; - else if (listdata[c] == '\n') newline = "\n"; - // Use CRLF as failsafe if .list file contains no newlines. - else newline = "\r\n"; - - // separate listdata into series of lines & newlines - size_t spn = 0; - for (char *c = listdata; *c; c += spn) - { endlines.push_back(""); - spn = strcspn(c, "\r\n"); - while (c[spn] == '\r' || c[spn] == '\n') - { endlines.back().push_back(c[spn]); - c[spn] = 0; - spn++; - } - lines.push_back(c); + // separate listdata into series of lines & newlines + size_t spn = 0; + for (char *c = listdata; *c; c += spn) + { endlines.push_back(""); + for (spn = strcspn(c, "\n\r"); c[spn] == '\n' || c[spn] == '\r'; spn++) + { endlines.back().push_back(c[spn]); + c[spn] = 0; } - lines.push_back(listdata+listdatasize+1); // add a dummy "past-the-end" element to make lines[l+1]-2 work - // strip UTF-8 byte order mark if present - if (!strncmp(lines[0], "\xEF\xBB\xBF", 3)) lines[0] += 3; - - for (unsigned int l = 0; l < lines.size()-1; l++) - { std::string orig_line(lines[l]); - // strip whitespace - while (lines[l][0] == ' ' || lines[l][0] == '\t') lines[l]++; - char * endchar = lines[l+1]-2; // -2 skips over the 0 inserted while separating listdata into lines - while (*endchar == 0) endchar--; // skip back more for CRLF cases, and lines followed by blank lines - while (*endchar == ' ' || *endchar == '\t') - { *endchar = 0; - endchar--; - } - std::string trim_line(lines[l]); - // ignore empty or "comment" lines - if (lines[l][0] == 0 || lines[l][0] == '#') - { splist << orig_line << endlines[l]; - continue; - } - // process fields in line - std::vector fields; - strtok_mtx->lock(); - for (char *token = strtok(lines[l], " \t"); token; token = strtok(0, " \t") ) fields.push_back(token); - strtok_mtx->unlock(); - if (fields.size() != 4) - // OK if 5th field exists and starts with # - if (fields.size() < 5 || fields[4][0] != '#') - { log << "Incorrect format line: " << trim_line << '\n'; - splist << orig_line << endlines[l]; - continue; - } + lines.push_back(c); + } + lines.push_back(listdata+listdatasize+1); // add a dummy "past-the-end" element to make lines[l+1]-2 work + // strip UTF-8 byte order mark if present + if (!strncmp(lines[0], "\xEF\xBB\xBF", 3)) lines[0] += 3; - // find the root that matches in some system and when we do, match labels - std::string route_entry = upper(std::string(fields[1])); // leave fields[1] intact for potential AltRouteName note - std::string lookup = std::string(upper(fields[0])) + " " + route_entry; - try { Route *r = Route::list_hash.at(lookup); - for (std::string& a : r->alt_route_names) - if (route_entry == a) - { log << "Note: deprecated route name " << fields[1] - << " -> canonical name " << r->list_entry_name() << " in line " << trim_line << '\n'; - break; - } - if (r->system->devel()) - { log << "Ignoring line matching highway in system in development: " << trim_line << '\n'; - splist << orig_line << endlines[l]; - continue; - } - // r is a route match, r.root is our root, and we need to find - // canonical waypoint labels, ignoring case and leading - // "+" or "*" when matching - std::vector point_indices; - unsigned int checking_index = 0; - while (*fields[2] == '*' || *fields[2] == '+') fields[2]++; - while (*fields[3] == '*' || *fields[3] == '+') fields[3]++; - upper(fields[2]); - upper(fields[3]); - for (Waypoint *w : r->point_list) - { std::string upper_label = upper(w->label); - while (upper_label.front() == '+' || upper_label.front() == '*') upper_label = upper_label.substr(1); - if (fields[2] == upper_label || fields[3] == upper_label) - { point_indices.push_back(checking_index); - r->liu_mtx.lock(); - r->labels_in_use.insert(upper_label); - r->liu_mtx.unlock(); - } - else { for (std::string &alt : w->alt_labels) - { if (fields[2] == alt || fields[3] == alt) - { point_indices.push_back(checking_index); - r->liu_mtx.lock(); - r->labels_in_use.insert(alt); - r->liu_mtx.unlock(); - // if we have not yet used this alt label, remove it from the unused set - r->ual_mtx.lock(); - r->unused_alt_labels.erase(alt); - r->ual_mtx.unlock(); - } - } - } - checking_index++; - } - if (point_indices.size() != 2) - { bool invalid_char = 0; - for (char& c : trim_line) - if (iscntrl(c)) - { c = '?'; - invalid_char = 1; - } - log << "Waypoint label(s) not found in line: " << trim_line; - if (invalid_char) log << " [contains invalid character(s)]"; - log << '\n'; - splist << orig_line << endlines[l]; - } - else { list_entries++; - // find the segments we just matched and store this traveler with the - // segments and the segments with the traveler (might not need both - // ultimately) - for (unsigned int wp_pos = point_indices[0]; wp_pos < point_indices[1]; wp_pos++) - { HighwaySegment *hs = r->segment_list[wp_pos]; - hs->add_clinched_by(this); - clinched_segments.insert(hs); - } - #include "splitregion.cpp" - } - } - catch (const std::out_of_range& oor) - { bool invalid_char = 0; - for (char& c : trim_line) - if (iscntrl(c)) - { c = '?'; - invalid_char = 1; - } - log << "Unknown region/highway combo in line: " << trim_line; - if (invalid_char) log << " [contains invalid character(s)]"; - log << '\n'; - splist << orig_line << endlines[l]; - } + for (unsigned int l = 0; l < lines.size()-1; l++) + { std::string orig_line(lines[l]); + // strip whitespace + while (lines[l][0] == ' ' || lines[l][0] == '\t') lines[l]++; + char * endchar = lines[l+1]-2; // -2 skips over the 0 inserted while separating listdata into lines + while (*endchar == 0) endchar--; // skip back more for CRLF cases, and lines followed by blank lines + while (*endchar == ' ' || *endchar == '\t') + { *endchar = 0; + endchar--; + } + std::string trim_line(lines[l]); + // ignore empty or "comment" lines + if (lines[l][0] == 0 || lines[l][0] == '#') + { splist << orig_line << endlines[l]; + continue; + } + // process fields in line + std::vector fields; + size_t spn = 0; + for (char* c = lines[l]; *c; c += spn) + { for (spn = strcspn(c, " \t"); c[spn] == ' ' || c[spn] == '\t'; spn++) c[spn] = 0; + if (*c == '#') break; + else fields.push_back(c); } - delete[] listdata; - log << "Processed " << list_entries << " good lines marking " << clinched_segments.size() << " segments traveled.\n"; - log.close(); - splist.close(); + if (fields.size() == 4) + { + #include "mark_chopped_route_segments.cpp" + } + else { log << "Incorrect format line: " << trim_line << '\n'; + splist << orig_line << endlines[l]; + } } + delete[] listdata; + log << "Processed " << list_entries << " good lines marking " << clinched_segments.size() << " segments traveled.\n"; + log.close(); + splist.close(); +} - /* Return active mileage across all regions */ - double active_only_miles() - { double mi = 0; - for (std::pair& rm : active_only_mileage_by_region) mi += rm.second; - return mi; - } +/* Return active mileage across all regions */ +double TravelerList::active_only_miles() +{ double mi = 0; + for (std::pair& rm : active_only_mileage_by_region) mi += rm.second; + return mi; +} - /* Return active+preview mileage across all regions */ - double active_preview_miles() - { double mi = 0; - for (std::pair& rm : active_preview_mileage_by_region) mi += rm.second; - return mi; - } +/* Return active+preview mileage across all regions */ +double TravelerList::active_preview_miles() +{ double mi = 0; + for (std::pair& rm : active_preview_mileage_by_region) mi += rm.second; + return mi; +} - /* Return mileage across all regions for a specified system */ - double system_region_miles(HighwaySystem *h) - { double mi = 0; - for (std::pair& rm : system_region_mileages.at(h)) mi += rm.second; - return mi; - } +/* Return mileage across all regions for a specified system */ +double TravelerList::system_region_miles(HighwaySystem *h) +{ double mi = 0; + for (std::pair& rm : system_region_mileages.at(h)) mi += rm.second; + return mi; +} - #include "userlog.cpp" -}; +#include "userlog.cpp" std::mutex TravelerList::alltrav_mtx; diff --git a/siteupdate/cplusplus/classes/TravelerList/TravelerList.h b/siteupdate/cplusplus/classes/TravelerList/TravelerList.h new file mode 100644 index 00000000..9bbceb10 --- /dev/null +++ b/siteupdate/cplusplus/classes/TravelerList/TravelerList.h @@ -0,0 +1,37 @@ +class TravelerList +{ /* This class encapsulates the contents of one .list file + that represents the travels of one individual user. + + A list file consists of lines of 4 values: + region route_name start_waypoint end_waypoint + + which indicates that the user has traveled the highway names + route_name in the given region between the waypoints named + start_waypoint end_waypoint + */ + public: + std::mutex ap_mi_mtx, ao_mi_mtx, sr_mi_mtx; + std::unordered_set clinched_segments; + std::string traveler_name; + std::unordered_map active_preview_mileage_by_region; // total mileage per region, active+preview only + std::unordered_map active_only_mileage_by_region; // total mileage per region, active only + std::unordered_map> system_region_mileages; // mileage per region per system + std::unordered_map> con_routes_traveled; // mileage per ConRte per system + // TODO is this necessary? + // ConRtes by definition exist in one system only + std::unordered_map routes_traveled; // mileage per traveled route + std::unordered_map con_routes_clinched; // clinch count per system + //std::unordered_map routes_clinched; // commented out in original siteupdate.py + unsigned int *traveler_num; + unsigned int active_systems_traveled; + unsigned int active_systems_clinched; + unsigned int preview_systems_traveled; + unsigned int preview_systems_clinched; + static std::mutex alltrav_mtx; // for locking the traveler_lists list when reading .lists from disk + + TravelerList(std::string, ErrorList *, Arguments *); + double active_only_miles(); + double active_preview_miles(); + double system_region_miles(HighwaySystem *); + void userlog(ClinchedDBValues *, const double, const double, std::list*, std::string path); +}; diff --git a/siteupdate/cplusplus/classes/TravelerList/mark_chopped_route_segments.cpp b/siteupdate/cplusplus/classes/TravelerList/mark_chopped_route_segments.cpp new file mode 100644 index 00000000..0ebed9a1 --- /dev/null +++ b/siteupdate/cplusplus/classes/TravelerList/mark_chopped_route_segments.cpp @@ -0,0 +1,111 @@ +// find the route that matches and when we do, match labels +std::string lookup = upper(fields[0]) + ( ' ' + upper(std::string(fields[1])) ); // leave fields[1] intact for potential AltRouteName note +// look for region/route combo, first in pri_list_hash +std::unordered_map::iterator rit = Route::pri_list_hash.find(lookup); +// and then if not found, in alt_list_hash +if (rit == Route::pri_list_hash.end()) +{ rit = Route::alt_list_hash.find(lookup); + if (rit == Route::alt_list_hash.end()) + { bool invalid_char = 0; + for (char& c : trim_line) + if (iscntrl(c)) + { c = '?'; + invalid_char = 1; + } + log << "Unknown region/highway combo in line: " << trim_line; + if (invalid_char) log << " [contains invalid character(s)]"; + log << '\n'; + splist << orig_line << endlines[l]; + continue; + } + else log << "Note: deprecated route name " << fields[1] + << " -> canonical name " << rit->second->list_entry_name() << " in line " << trim_line << '\n'; +} +Route* r = rit->second; +if (r->system->devel()) +{ log << "Ignoring line matching highway in system in development: " << trim_line << '\n'; + splist << orig_line << endlines[l]; + continue; +} +// r is a route match, and we need to find +// waypoint indices, ignoring case and leading +// '+' or '*' when matching +unsigned int index1, index2; +while (*fields[2] == '*' || *fields[2] == '+') fields[2]++; +while (*fields[3] == '*' || *fields[3] == '+') fields[3]++; +upper(fields[2]); +upper(fields[3]); + +// look for point indices for labels, first in pri_label_hash +std::unordered_map::iterator lit1 = r->pri_label_hash.find(fields[2]); +std::unordered_map::iterator lit2 = r->pri_label_hash.find(fields[3]); +// and then if not found, in alt_label_hash +if (lit1 == r->pri_label_hash.end()) lit1 = r->alt_label_hash.find(fields[2]); +if (lit2 == r->pri_label_hash.end()) lit2 = r->alt_label_hash.find(fields[3]); + +// if we did not find matches for both labels... +if (lit1 == r->alt_label_hash.end() || lit2 == r->alt_label_hash.end()) +{ bool invalid_char = 0; + for (char& c : trim_line) + if (iscntrl(c)) + { c = '?'; + invalid_char = 1; + } + for (char* c = fields[2]; *c; c++) if (iscntrl(*c)) *c = '?'; + for (char* c = fields[3]; *c; c++) if (iscntrl(*c)) *c = '?'; + if (lit1 == lit2) + log << "Waypoint labels " << fields[2] << " and " << fields[3] << " not found in line: " << trim_line; + else { log << "Waypoint label "; + log << (lit1 == r->alt_label_hash.end() ? fields[2] : fields[3]); + log << " not found in line: " << trim_line; + } + if (invalid_char) log << " [contains invalid character(s)]"; + log << '\n'; + splist << orig_line << endlines[l]; + continue; +} +// are either of the labels used duplicates? +char duplicate = 0; +if (r->duplicate_labels.find(fields[2]) != r->duplicate_labels.end()) +{ log << r->region->code << ": duplicate label " << fields[2] << " in " << r->root + << ". Please report this error in the TravelMapping forum" + << ". Unable to parse line: " << trim_line << '\n'; + duplicate = 1; +} +if (r->duplicate_labels.find(fields[3]) != r->duplicate_labels.end()) +{ log << r->region->code << ": duplicate label " << fields[3] << " in " << r->root + << ". Please report this error in the TravelMapping forum" + << ". Unable to parse line: " << trim_line << '\n'; + duplicate = 1; +} +if (duplicate) continue; +// if both labels reference the same waypoint... +if (lit1->second == lit2->second) + log << "Equivalent waypoint labels mark zero distance traveled in line: " << trim_line << '\n'; +// otherwise both labels are valid; mark in use & proceed +else { r->system->lniu_mtx.lock(); + r->system->listnamesinuse.insert(lookup); + r->system->lniu_mtx.unlock(); + r->system->uarn_mtx.lock(); + r->system->unusedaltroutenames.erase(lookup); + r->system->uarn_mtx.unlock(); + r->liu_mtx.lock(); + r->labels_in_use.insert(fields[2]); + r->labels_in_use.insert(fields[3]); + r->liu_mtx.unlock(); + r->ual_mtx.lock(); + r->unused_alt_labels.erase(fields[2]); + r->unused_alt_labels.erase(fields[3]); + r->ual_mtx.unlock(); + + list_entries++; + if (lit1->second < lit2->second) + { index1 = lit1->second; + index2 = lit2->second; + } + else { index1 = lit2->second; + index2 = lit1->second; + } + r->store_traveled_segments(this, index1, index2); + #include "splitregion.cpp" + } diff --git a/siteupdate/cplusplus/classes/TravelerList/splitregion.cpp b/siteupdate/cplusplus/classes/TravelerList/splitregion.cpp index 07ab5859..a3f967bd 100644 --- a/siteupdate/cplusplus/classes/TravelerList/splitregion.cpp +++ b/siteupdate/cplusplus/classes/TravelerList/splitregion.cpp @@ -2,7 +2,7 @@ if (args->splitregion == upper(fields[0])) { // first, comment out original line splist << "##### " << orig_line << newline; - HighwaySegment *orig_hs = r->segment_list[point_indices[0]]; + HighwaySegment *orig_hs = r->segment_list[index1]; HighwaySegment *new_hs = 0; if (!orig_hs->concurrent) std::cout << "ERROR: " << orig_hs->str() << " not concurrent" << std::endl; @@ -17,7 +17,7 @@ if (args->splitregion == upper(fields[0])) else { if (count > 1) std::cout << "DEBUG: multiple matches found for " << orig_hs->str() << std::endl; // get lines from associated connected route. // assumption: each chopped route in old full region corresponds 1:1 to a connected route in new chopped regions - splist << new_hs->route->con_route->list_lines(point_indices[0], point_indices[1] - point_indices[0], newline, 2) << endlines[l]; + splist << new_hs->route->con_route->list_lines(index1, index2 - index1, newline, 2) << endlines[l]; } } } diff --git a/siteupdate/cplusplus/classes/TravelerList/userlog.cpp b/siteupdate/cplusplus/classes/TravelerList/userlog.cpp index 1ea1417e..5e35dfa3 100644 --- a/siteupdate/cplusplus/classes/TravelerList/userlog.cpp +++ b/siteupdate/cplusplus/classes/TravelerList/userlog.cpp @@ -1,4 +1,4 @@ -void userlog +void TravelerList::userlog ( ClinchedDBValues *clin_db_val, const double total_active_only_miles, const double total_active_preview_miles, diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp index 1b84ac51..3eea81ef 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp @@ -20,11 +20,7 @@ Waypoint::Waypoint(char *line, Route *rte, DatacheckEntryList *datacheckerrors) // parse WPT line size_t spn = 0; for (char* c = line; *c; c += spn) - { spn = strcspn(c, " "); - while (c[spn] == ' ') - { c[spn] = 0; - spn++; - } + { for (spn = strcspn(c, " "); c[spn] == ' '; spn++) c[spn] = 0; alt_labels.emplace_back(c); } @@ -309,19 +305,6 @@ bool Waypoint::label_references_route(Route *r, DatacheckEntryList *datacheckerr /* Datacheck */ -inline void Waypoint::duplicate_label(DatacheckEntryList *datacheckerrors, std::unordered_set &all_route_labels) -{ // duplicate labels - // first, check primary label - std::string upper_label = upper(label); - while (upper_label[0] == '+' || upper_label[0] == '*') upper_label = upper_label.substr(1); - if (!all_route_labels.insert(upper_label).second) - datacheckerrors->add(route, upper_label, "", "", "DUPLICATE_LABEL", ""); - // then check alt labels - for (std::string &a : alt_labels) - if (!all_route_labels.insert(a).second) - datacheckerrors->add(route, a, "", "", "DUPLICATE_LABEL", ""); -} - inline void Waypoint::duplicate_coords(DatacheckEntryList *datacheckerrors, std::unordered_set &coords_used, char *fstr) { // duplicate coordinates Waypoint *w; diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.h b/siteupdate/cplusplus/classes/Waypoint/Waypoint.h index e361800a..7e302c8c 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.h +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.h @@ -39,7 +39,6 @@ class Waypoint bool label_references_route(Route *, DatacheckEntryList *); // Datacheck - inline void duplicate_label(DatacheckEntryList *, std::unordered_set &); inline void duplicate_coords(DatacheckEntryList *, std::unordered_set &, char *); inline void out_of_bounds(DatacheckEntryList *, char *); inline void distance_update(DatacheckEntryList *, char *, double &, Waypoint *); diff --git a/siteupdate/cplusplus/classes/WaypointQuadtree/WaypointQuadtree.cpp b/siteupdate/cplusplus/classes/WaypointQuadtree/WaypointQuadtree.cpp index e6e86243..4d3f8bb5 100644 --- a/siteupdate/cplusplus/classes/WaypointQuadtree/WaypointQuadtree.cpp +++ b/siteupdate/cplusplus/classes/WaypointQuadtree/WaypointQuadtree.cpp @@ -1,4 +1,4 @@ -bool WaypointQuadtree::WaypointQuadtree::refined() +inline bool WaypointQuadtree::WaypointQuadtree::refined() { return nw_child; } diff --git a/siteupdate/cplusplus/classes/WaypointQuadtree/WaypointQuadtree.h b/siteupdate/cplusplus/classes/WaypointQuadtree/WaypointQuadtree.h index 87da2852..7bba1137 100644 --- a/siteupdate/cplusplus/classes/WaypointQuadtree/WaypointQuadtree.h +++ b/siteupdate/cplusplus/classes/WaypointQuadtree/WaypointQuadtree.h @@ -9,7 +9,7 @@ class WaypointQuadtree unsigned int unique_locations; std::recursive_mutex mtx; - bool refined(); + inline bool refined(); WaypointQuadtree(double, double, double, double); void refine(); void insert(Waypoint*, bool); diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index 7ab1b1d1..8b7c8ce4 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -44,11 +44,14 @@ class HGEdge; #include #include #include +#include "classes/Arguments.cpp" +#include "classes/ClinchedDBValues.cpp" #include "classes/WaypointQuadtree/WaypointQuadtree.h" #include "classes/Route/Route.h" #include "classes/ConnectedRoute/ConnectedRoute.h" #include "classes/Waypoint/Waypoint.h" #include "classes/HighwaySegment/HighwaySegment.h" +#include "classes/TravelerList/TravelerList.h" #include "classes/GraphGeneration/HGEdge.h" #include "classes/GraphGeneration/PlaceRadius.h" #include "enable_threading.cpp" @@ -62,7 +65,6 @@ class HGEdge; #include "functions/split.cpp" #include "functions/valid_num_str.cpp" #include "classes/DBFieldLength.cpp" -#include "classes/Arguments.cpp" #include "classes/ElapsedTime.cpp" #include "classes/ErrorList.cpp" #include "classes/Region.cpp" @@ -71,7 +73,6 @@ class HGEdge; #include "classes/DatacheckEntryList.cpp" #include "classes/Waypoint/Waypoint.cpp" #include "classes/WaypointQuadtree/WaypointQuadtree.cpp" -#include "classes/ClinchedDBValues.cpp" #include "classes/Route/Route.cpp" #include "classes/ConnectedRoute/ConnectedRoute.cpp" #include "classes/TravelerList/TravelerList.cpp" @@ -96,7 +97,7 @@ using namespace std; int main(int argc, char *argv[]) { ifstream file; string line; - mutex list_mtx, log_mtx, strtok_mtx; + mutex list_mtx, log_mtx; time_t timestamp; // start a timer for including elapsed time reports in messages @@ -384,17 +385,45 @@ int main(int argc, char *argv[]) #include "functions/concurrency_detection.cpp" - cout << et.et() << "Checking for unconnected chopped routes, creating canonical AltLabels & populating unused sets." << endl; + cout << et.et() << "Processing waypoint labels and checking for unconnected chopped routes." << endl; for (HighwaySystem* h : highway_systems) for (Route& r : h->route_list) - { if (!r.con_route) - el.add_error(r.system->systemname + ".csv: root " + r.root + " not matched by any connected route root."); - for (Waypoint* w : r.point_list) - for (std::string& a : w->alt_labels) - { while (a[0] == '+' || a[0] == '*') a = a.substr(1); - upper(a.data()); - r.unused_alt_labels.insert(a); - } + { // check for unconnected chopped routes + if (!r.con_route) + el.add_error(r.system->systemname + ".csv: root " + r.root + " not matched by any connected route root."); + unsigned int index = 0; + for (Waypoint* w : r.point_list) + { // ignore case and leading '+' or '*' + std::string upper_label = upper(w->label); + while (upper_label[0] == '+' || upper_label[0] == '*') + upper_label = upper_label.substr(1); + // if primary label not duplicated, add to r.pri_label_hash + if (r.alt_label_hash.find(upper_label) != r.alt_label_hash.end()) + { datacheckerrors->add(&r, upper_label, "", "", "DUPLICATE_LABEL", ""); + r.duplicate_labels.insert(upper_label); + } + else if (!r.pri_label_hash.insert(std::pair(upper_label, index)).second) + { datacheckerrors->add(&r, upper_label, "", "", "DUPLICATE_LABEL", ""); + r.duplicate_labels.insert(upper_label); + } + for (std::string& a : w->alt_labels) + { // create canonical AltLabels + while (a[0] == '+' || a[0] == '*') a = a.substr(1); + upper(a.data()); + // populate unused set + r.unused_alt_labels.insert(a); + // create label->index hashes and check if AltLabels duplicated + if (r.pri_label_hash.find(a) != r.pri_label_hash.end()) + { datacheckerrors->add(&r, a, "", "", "DUPLICATE_LABEL", ""); + r.duplicate_labels.insert(a); + } + else if (!r.alt_label_hash.insert(std::pair(a, index)).second) + { datacheckerrors->add(&r, a, "", "", "DUPLICATE_LABEL", ""); + r.duplicate_labels.insert(a); + } + } + index++; + } } // Create a list of TravelerList objects, one per person @@ -405,7 +434,7 @@ int main(int argc, char *argv[]) #ifdef threading_enabled // set up for threaded .list file processing for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(ReadListThread, t, &traveler_ids, &id_it, &traveler_lists, &list_mtx, &strtok_mtx, &args, &el); + thr[t] = new thread(ReadListThread, t, &traveler_ids, &id_it, &traveler_lists, &list_mtx, &args, &el); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); for (unsigned int t = 0; t < args.numthreads; t++) @@ -413,7 +442,7 @@ int main(int argc, char *argv[]) #else for (string &t : traveler_ids) { cout << t << ' ' << std::flush; - traveler_lists.push_back(new TravelerList(t, &el, &args, &strtok_mtx)); + traveler_lists.push_back(new TravelerList(t, &el, &args)); } traveler_ids.clear(); #endif @@ -426,8 +455,16 @@ int main(int argc, char *argv[]) travnum++; } - //#include "debug/highway_segment_log.cpp" - //#include "debug/pioneers.cpp" + cout << et.et() << "Clearing route & label hash tables." << endl; + Route::root_hash.clear(); + Route::pri_list_hash.clear(); + Route::alt_list_hash.clear(); + for (HighwaySystem* h : highway_systems) + for (Route& r : h->route_list) + { r.pri_label_hash.clear(); + r.alt_label_hash.clear(); + r.duplicate_labels.clear(); + } // Read updates.csv file, just keep in the fields list for now since we're // just going to drop this into the DB later anyway @@ -516,47 +553,72 @@ int main(int argc, char *argv[]) } file.close(); - // write log file for points in use -- might be more useful in the DB later, - // or maybe in another format - cout << et.et() << "Writing points in use log." << endl; - ofstream inusefile(args.logfilepath+"/pointsinuse.log"); - timestamp = time(0); - inusefile << "Log file created at: " << ctime(×tamp); - for (HighwaySystem *h : highway_systems) - for (Route &r : h->route_list) - if (r.labels_in_use.size()) - { inusefile << r.root << '(' << r.point_list.size() << "):"; - list liu_list(r.labels_in_use.begin(), r.labels_in_use.end()); - liu_list.sort(); - for (string& label : liu_list) inusefile << ' ' << label; - inusefile << '\n'; - r.labels_in_use.clear(); - } - inusefile.close(); - - // write log file for alt labels not in use - cout << et.et() << "Writing unused alt labels log." << endl; + cout << et.et() << "Writing pointsinuse.log, unusedaltlabels.log, listnamesinuse.log and unusedaltroutenames.log" << endl; unsigned int total_unused_alt_labels = 0; + unsigned int total_unusedaltroutenames = 0; list unused_alt_labels; + ofstream piufile(args.logfilepath+"/pointsinuse.log"); + ofstream lniufile(args.logfilepath+"/listnamesinuse.log"); + ofstream uarnfile(args.logfilepath+"/unusedaltroutenames.log"); + timestamp = time(0); + piufile << "Log file created at: " << ctime(×tamp); + lniufile << "Log file created at: " << ctime(×tamp); + uarnfile << "Log file created at: " << ctime(×tamp); for (HighwaySystem *h : highway_systems) - for (Route &r : h->route_list) + { for (Route &r : h->route_list) + { // labelsinuse.log line + if (r.labels_in_use.size()) + { piufile << r.root << '(' << r.point_list.size() << "):"; + list liu_list(r.labels_in_use.begin(), r.labels_in_use.end()); + liu_list.sort(); + for (string& label : liu_list) piufile << ' ' << label; + piufile << '\n'; + r.labels_in_use.clear(); + } + // unusedaltlabels.log lines, to be sorted by root later if (r.unused_alt_labels.size()) { total_unused_alt_labels += r.unused_alt_labels.size(); string ual_entry = r.root + '(' + to_string(r.unused_alt_labels.size()) + "):"; list ual_list(r.unused_alt_labels.begin(), r.unused_alt_labels.end()); ual_list.sort(); for (string& label : ual_list) ual_entry += ' ' + label; - r.unused_alt_labels.clear(); unused_alt_labels.push_back(ual_entry); + r.unused_alt_labels.clear(); } + } + // listnamesinuse.log line + if (h->listnamesinuse.size()) + { lniufile << h->systemname << '(' << h->route_list.size() << "):"; + list lniu_list(h->listnamesinuse.begin(), h->listnamesinuse.end()); + lniu_list.sort(); + for (string& list_name : lniu_list) lniufile << " \"" << list_name << '"'; + lniufile << '\n'; + h->listnamesinuse.clear(); + } + // unusedaltroutenames.log line + if (h->unusedaltroutenames.size()) + { total_unusedaltroutenames += h->unusedaltroutenames.size(); + uarnfile << h->systemname << '(' << h->unusedaltroutenames.size() << "):"; + list uarn_list(h->unusedaltroutenames.begin(), h->unusedaltroutenames.end()); + uarn_list.sort(); + for (string& list_name : uarn_list) uarnfile << " \"" << list_name << '"'; + uarnfile << '\n'; + h->unusedaltroutenames.clear(); + } + } + piufile.close(); + lniufile.close(); + uarnfile << "Total: " << total_unusedaltroutenames << '\n'; + uarnfile.close(); + // sort lines and write unusedaltlabels.log unused_alt_labels.sort(); - ofstream unusedfile(args.logfilepath+"/unusedaltlabels.log"); + ofstream ualfile(args.logfilepath+"/unusedaltlabels.log"); timestamp = time(0); - unusedfile << "Log file created at: " << ctime(×tamp); - for (string &ual_entry : unused_alt_labels) unusedfile << ual_entry << '\n'; + ualfile << "Log file created at: " << ctime(×tamp); + for (string &ual_entry : unused_alt_labels) ualfile << ual_entry << '\n'; unused_alt_labels.clear(); - unusedfile << "Total: " << total_unused_alt_labels << '\n'; - unusedfile.close(); + ualfile << "Total: " << total_unused_alt_labels << '\n'; + ualfile.close(); // now augment any traveler clinched segments for concurrencies diff --git a/siteupdate/cplusplus/threads/ReadListThread.cpp b/siteupdate/cplusplus/threads/ReadListThread.cpp index 923fb45b..f48f1af9 100644 --- a/siteupdate/cplusplus/threads/ReadListThread.cpp +++ b/siteupdate/cplusplus/threads/ReadListThread.cpp @@ -1,7 +1,6 @@ void ReadListThread ( unsigned int id, std::list *traveler_ids, std::list::iterator *it, - std::list *traveler_lists, std::mutex *tl_mtx, std::mutex *strtok_mtx, - Arguments *args, ErrorList *el + std::list *traveler_lists, std::mutex *tl_mtx, Arguments *args, ErrorList *el ) { //printf("Starting ReadListThread %02i\n", id); fflush(stdout); while (*it != traveler_ids->end()) @@ -16,7 +15,7 @@ void ReadListThread //printf("ReadListThread %02i (*it)++\n", id); fflush(stdout); std::cout << tl << ' ' << std::flush; tl_mtx->unlock(); - TravelerList *t = new TravelerList(tl, el, args, strtok_mtx); + TravelerList *t = new TravelerList(tl, el, args); // deleted on termination of program TravelerList::alltrav_mtx.lock(); traveler_lists->push_back(t); diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 873ada68..66472c33 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -756,13 +756,13 @@ def label_references_route(self, r, datacheckerrors): return True if self.label[len(no_abbrev):len(no_abbrev)+len(r.abbrev)] != r.abbrev: #if self.label[len(no_abbrev)] == '/': - #datacheckerrors.append(route, [self.label], "UNEXPECTED_DESIGNATION", self.label[len(no_abbrev)+1:]) + #datacheckerrors.append(DatacheckEntry(route, [self.label], "UNEXPECTED_DESIGNATION", self.label[len(no_abbrev)+1:])) return False if len(self.label) == len(no_abbrev) + len(r.abbrev) \ or self.label[len(no_abbrev) + len(r.abbrev)] == '_': return True #if self.label[len(no_abbrev) + len(r.abbrev)] == '/': - #datacheckerrors.append(route, [self.label], "UNEXPECTED_DESIGNATION", self.label[len(no_abbrev)+len(r.abbrev)+1:]) + #datacheckerrors.append(DatacheckEntry(route, [self.label], "UNEXPECTED_DESIGNATION", self.label[len(no_abbrev)+len(r.abbrev)+1:])) return False def label_too_long(self, datacheckerrors): @@ -895,7 +895,8 @@ class Route: alternate route names that might appear in user list files. """ root_hash = dict() - list_hash = dict() + pri_list_hash = dict() + alt_list_hash = dict() def __init__(self,line,system,el): """initialize object from a .csv file line, but do not @@ -955,25 +956,36 @@ def __init__(self,line,system,el): " already in " + Route.root_hash[self.root].system.systemname + ".csv") else: Route.root_hash[self.root] = self - # insert into list_hash with list name, checking for duplicate .list names + # insert list name into pri_list_hash, checking for duplicate .list names list_name = self.readable_name().upper() - if list_name in Route.list_hash: + if list_name in Route.alt_list_hash: el.add_error("Duplicate main list name in " + self.root + ": '" + self.readable_name() + - "' already points to " + Route.list_hash[list_name].root) + "' already points to " + Route.alt_list_hash[list_name].root) + elif list_name in Route.pri_list_hash: + el.add_error("Duplicate main list name in " + self.root + ": '" + self.readable_name() + + "' already points to " + Route.pri_list_hash[list_name].root) else: - Route.list_hash[list_name] = self - # insert into list_hash with alt names, checking for duplicate .list names + Route.pri_list_hash[list_name] = self + # insert alt names into alt_list_hash, checking for duplicate .list names for a in self.alt_route_names: list_name = self.region.upper() + ' ' + a - if list_name in Route.list_hash: + if list_name in Route.pri_list_hash: + el.add_error("Duplicate alt route name in " + self.root + ": '" + self.region + ' ' + a + + "' already points to " + Route.pri_list_hash[list_name].root) + elif list_name in Route.alt_list_hash: el.add_error("Duplicate alt route name in " + self.root + ": '" + self.region + ' ' + a + - "' already points to " + Route.list_hash[list_name].root) + "' already points to " + Route.alt_list_hash[list_name].root) else: - Route.list_hash[list_name] = self + Route.alt_list_hash[list_name] = self + # populate unused set + system.unusedaltroutenames.add(list_name) self.point_list = [] self.labels_in_use = set() self.unused_alt_labels = set() + self.duplicate_labels = set() + self.pri_label_hash = dict() + self.alt_label_hash = dict() self.segment_list = [] self.con_route = None self.mileage = 0.0 @@ -1076,6 +1088,13 @@ def clinched_by_traveler(self,t): miles += s.length return miles + def store_traveled_segments(self, t, beg, end): + # store clinched segments with traveler and traveler with segments + for pos in range(beg, end): + hs = self.segment_list[pos] + hs.add_clinched_by(t) + t.clinched_segments.add(hs) + class ConnectedRoute: """This class encapsulates a single 'connected route' as given by a single line of a _con.csv file @@ -1175,6 +1194,8 @@ def __init__(self,systemname,country,fullname,color,tier,level,el, self.level = level self.mileage_by_region = dict() self.vertices = set() + self.listnamesinuse = set() + self.unusedaltroutenames = set() # read chopped routes .csv try: @@ -1262,65 +1283,103 @@ def __init__(self,travelername,el,path="../../../UserData/list_files"): if len(line) == 0 or line.startswith("#"): continue fields = re.split('[ \t]+',line) - if len(fields) != 4: - # OK if 5th field exists and starts with # - if len(fields) < 5 or not fields[4].startswith("#"): - self.log_entries.append("Incorrect format line: " + line) - continue - - # find the root that matches in some system and when we do, match labels - route_entry = fields[1].upper() - lookup = fields[0].upper() + ' ' + route_entry - if lookup not in Route.list_hash: - (line, invchar) = no_control_chars(line) - self.log_entries.append("Unknown region/highway combo in line: " + line) - if invchar: - self.log_entries[-1] += " [contains invalid character(s)]" - else: - r = Route.list_hash[lookup] - for a in r.alt_route_names: - if route_entry == a: + # truncate inline comments from fields list + for i in range(min(len(fields), 5)): + if fields[i][0] == '#': + fields = fields[:i] + break + if len(fields) == 4: + # find the route that matches and when we do, match labels + lookup = fields[0].upper() + ' ' + fields[1].upper() + # look for region/route combo, first in pri_list_hash + try: + r = Route.pri_list_hash[lookup] + # and then if not found, in alt_list_hash + except KeyError: + try: + r = Route.alt_list_hash[lookup] + except KeyError: + (line, invchar) = no_control_chars(line) + self.log_entries.append("Unknown region/highway combo in line: " + line) + if invchar: + self.log_entries[-1] += " [contains invalid character(s)]" + continue + else: self.log_entries.append("Note: deprecated route name " + fields[1] + " -> canonical name " + r.list_entry_name() + " in line " + line) - break - if r.system.devel(): self.log_entries.append("Ignoring line matching highway in system in development: " + line) continue - # r is a route match, r.root is our root, and we need to find - # canonical waypoint labels, ignoring case and leading + # r is a route match, and we need to find + # waypoint indices, ignoring case and leading # "+" or "*" when matching - point_indices = [] - checking_index = 0; list_label_1 = fields[2].lstrip("+*").upper() list_label_2 = fields[3].lstrip("+*").upper() - for w in r.point_list: - upper_label = w.label.lstrip("+*").upper() - if list_label_1 == upper_label or list_label_2 == upper_label: - point_indices.append(checking_index) - r.labels_in_use.add(upper_label) - else: - for alt in w.alt_labels: - if list_label_1 == alt or list_label_2 == alt: - point_indices.append(checking_index) - r.labels_in_use.add(alt) - r.unused_alt_labels.discard(alt) - - checking_index += 1 - if len(point_indices) != 2: + + # look for point indices for labels, first in pri_label_hash + # and then if not found, in alt_label_hash + try: + index1 = r.pri_label_hash[list_label_1] + except KeyError: + try: + index1 = r.alt_label_hash[list_label_1] + except KeyError: + index1 = None + try: + index2 = r.pri_label_hash[list_label_2] + except KeyError: + try: + index2 = r.alt_label_hash[list_label_2] + except KeyError: + index2 = None + # if we did not find matches for both labels... + if index1 is None or index2 is None: + (list_label_1, invchar) = no_control_chars(list_label_1) + (list_label_2, invchar) = no_control_chars(list_label_2) (line, invchar) = no_control_chars(line) - self.log_entries.append("Waypoint label(s) not found in line: " + line) + if index1 == index2: + self.log_entries.append("Waypoint labels " + list_label_1 + " and " \ + + list_label_2 + " not found in line: " + line) + else: + log_entry = "Waypoint label " + log_entry += list_label_1 if index1 is None else list_label_2 + log_entry += " not found in line: " + line + self.log_entries.append(log_entry) if invchar: self.log_entries[-1] += " [contains invalid character(s)]" + continue + # are either of the labels used duplicates? + duplicate = False + if list_label_1 in r.duplicate_labels: + log_entry = r.region + ": duplicate label " + list_label_1 + " in " + r.root \ + + ". Please report this error in the TravelMapping forum" \ + + ". Unable to parse line: " + line + self.log_entries.append(log_entry) + duplicate = True + if list_label_2 in r.duplicate_labels: + log_entry = r.region + ": duplicate label " + list_label_2 + " in " + r.root \ + + ". Please report this error in the TravelMapping forum" \ + + ". Unable to parse line: " + line + self.log_entries.append(log_entry) + duplicate = True + if duplicate: + continue + # if both labels reference the same waypoint... + if index1 == index2: + self.log_entries.append("Equivalent waypoint labels mark zero distance traveled in line: " + line) + # otherwise both labels are valid; mark in use & proceed else: + r.system.listnamesinuse.add(lookup) + r.system.unusedaltroutenames.discard(lookup) + r.labels_in_use.add(list_label_1) + r.labels_in_use.add(list_label_2) + r.unused_alt_labels.discard(list_label_1) + r.unused_alt_labels.discard(list_label_2) list_entries += 1 - # find the segments we just matched and store this traveler with the - # segments and the segments with the traveler (might not need both - # ultimately) - for wp_pos in range(point_indices[0],point_indices[1]): - hs = r.segment_list[wp_pos] - hs.add_clinched_by(self) - if hs not in self.clinched_segments: - self.clinched_segments.add(hs) + if index1 > index2: + (index1, index2) = (index2, index1) + r.store_traveled_segments(self, index1, index2) + else: + self.log_entries.append("Incorrect format line: " + line) self.log_entries.append("Processed " + str(list_entries) + \ " good lines marking " +str(len(self.clinched_segments)) + \ @@ -2681,15 +2740,40 @@ def run(self): print(".", end="", flush=True) print() -print(et.et() + "Checking for unconnected chopped routes, creating canonical AltLabels & populating unused sets.",flush=True) +print(et.et() + "Processing waypoint labels and checking for unconnected chopped routes.",flush=True) for h in highway_systems: for r in h.route_list: + # check for unconnected chopped routes if r.con_route is None: el.add_error(r.system.systemname + ".csv: root " + r.root + " not matched by any connected route root.") + index = 0 for w in r.point_list: + # ignore case and leading '+' or '*' + upper_label = w.label.lstrip('+*').upper() + # if primary label not duplicated, add to r.pri_label_hash + if upper_label in r.alt_label_hash: + datacheckerrors.append(DatacheckEntry(r, [upper_label], "DUPLICATE_LABEL")) + r.duplicate_labels.add(upper_label) + elif upper_label in r.pri_label_hash: + datacheckerrors.append(DatacheckEntry(r, [upper_label], "DUPLICATE_LABEL")) + r.duplicate_labels.add(upper_label) + else: + r.pri_label_hash[upper_label] = index for a in range(len(w.alt_labels)): + # create canonical AltLabels w.alt_labels[a] = w.alt_labels[a].lstrip('+*').upper() + # populate unused set r.unused_alt_labels.add(w.alt_labels[a]) + # create label->index hashes and check if AltLabels duplicated + if w.alt_labels[a] in r.pri_label_hash: + datacheckerrors.append(DatacheckEntry(r, [w.alt_labels[a]], "DUPLICATE_LABEL")) + r.duplicate_labels.add(a) + elif w.alt_labels[a] in r.alt_label_hash: + datacheckerrors.append(DatacheckEntry(r, [w.alt_labels[a]], "DUPLICATE_LABEL")) + r.duplicate_labels.add(a) + else: + r.alt_label_hash[w.alt_labels[a]] = index + index += 1 # Create a list of TravelerList objects, one per person traveler_lists = [] @@ -2707,6 +2791,16 @@ def run(self): t.traveler_num = travnum travnum += 1 +print(et.et() + "Clearing route & label hash tables.",flush=True) +del Route.root_hash +del Route.pri_list_hash +del Route.alt_list_hash +for h in highway_systems: + for r in h.route_list: + del r.pri_label_hash + del r.alt_label_hash + del r.duplicate_labels + # Read updates.csv file, just keep in the fields array for now since we're # just going to drop this into the DB later anyway updates = [] @@ -2782,42 +2876,61 @@ def run(self): systemupdates.append(fields) print("") -# write log file for points in use -- might be more useful in the DB later, -# or maybe in another format -print(et.et() + "Writing points in use log.") -inusefile = open(args.logfilepath+'/pointsinuse.log','w',encoding='UTF-8') -inusefile.write("Log file created at: " + str(datetime.datetime.now()) + "\n") -for h in highway_systems: - for r in h.route_list: - if len(r.labels_in_use) > 0: - inusefile.write(r.root + "(" + str(len(r.point_list)) + "):") - for label in sorted(r.labels_in_use): - inusefile.write(" " + label) - inusefile.write("\n") - r.labels_in_use = None -inusefile.close() - -# write log file for alt labels not in use -print(et.et() + "Writing unused alt labels log.") +print(et.et() + "Writing pointsinuse.log, unusedaltlabels.log, listnamesinuse.log and unusedaltroutenames.log",flush=True) total_unused_alt_labels = 0 +total_unusedaltroutenames = 0 unused_alt_labels = [] +piufile = open(args.logfilepath+'/pointsinuse.log','w',encoding='UTF-8') +lniufile = open(args.logfilepath+'/listnamesinuse.log','w',encoding='UTF-8') +uarnfile = open(args.logfilepath+'/unusedaltroutenames.log','w',encoding='UTF-8') +piufile.write("Log file created at: " + str(datetime.datetime.now()) + "\n") +lniufile.write("Log file created at: " + str(datetime.datetime.now()) + "\n") +uarnfile.write("Log file created at: " + str(datetime.datetime.now()) + "\n") for h in highway_systems: for r in h.route_list: + # labelsinuse.log line + if len(r.labels_in_use) > 0: + piufile.write(r.root + "(" + str(len(r.point_list)) + "):") + for label in sorted(r.labels_in_use): + piufile.write(" " + label) + piufile.write("\n") + del r.labels_in_use + # unusedaltlabels.log lines, to be sorted by root later if len(r.unused_alt_labels) > 0: total_unused_alt_labels += len(r.unused_alt_labels) ual_entry = r.root + "(" + str(len(r.unused_alt_labels)) + "):" for label in sorted(r.unused_alt_labels): ual_entry += " " + label - r.unused_alt_labels = None unused_alt_labels.append(ual_entry) + del r.unused_alt_labels + #listnamesinuse.log line + if len(h.listnamesinuse) > 0: + lniufile.write(h.systemname + '(' + str(len(h.route_list)) + "):") + for list_name in sorted(h.listnamesinuse): + lniufile.write(" \"" + list_name + '"') + lniufile.write('\n') + del h.listnamesinuse + # unusedaltroutenames.log line + if len(h.unusedaltroutenames) > 0: + total_unusedaltroutenames += len(h.unusedaltroutenames) + uarnfile.write(h.systemname + '(' + str(len(h.unusedaltroutenames)) + "):") + for list_name in sorted(h.unusedaltroutenames): + uarnfile.write(" \"" + list_name + '"') + uarnfile.write('\n') + del h.unusedaltroutenames +piufile.close() +lniufile.close() +uarnfile.write("Total: " + str(total_unusedaltroutenames) + '\n') +uarnfile.close() +# sort lines and write unusedaltlabels.log unused_alt_labels.sort() -unusedfile = open(args.logfilepath+'/unusedaltlabels.log','w',encoding='UTF-8') -unusedfile.write("Log file created at: " + str(datetime.datetime.now()) + "\n") +ualfile = open(args.logfilepath+'/unusedaltlabels.log','w',encoding='UTF-8') +ualfile.write("Log file created at: " + str(datetime.datetime.now()) + "\n") for ual_entry in unused_alt_labels: - unusedfile.write(ual_entry + "\n") + ualfile.write(ual_entry + "\n") unused_alt_labels = None -unusedfile.write("Total: " + str(total_unused_alt_labels) + "\n") -unusedfile.close() +ualfile.write("Total: " + str(total_unused_alt_labels) + "\n") +ualfile.close() # concurrency detection -- will augment our structure with list of concurrent @@ -3548,8 +3661,6 @@ def run(self): for h in highway_systems: print(".",end="",flush=True) for r in h.route_list: - # set to be used per-route to find label duplicates - all_route_labels = set() # set of tuples to be used for finding duplicate coordinates coords_used = set() @@ -3567,20 +3678,6 @@ def run(self): datacheckerrors.append(DatacheckEntry(r,[r.point_list[-1].label],'HIDDEN_TERMINUS')) for w in r.point_list: - # duplicate labels - # first, check primary label - upper_label = w.label.strip("+*").upper() - if upper_label in all_route_labels: - datacheckerrors.append(DatacheckEntry(r,[upper_label],"DUPLICATE_LABEL")) - else: - all_route_labels.add(upper_label) - # then check alt labels - for a in w.alt_labels: - if a in all_route_labels: - datacheckerrors.append(DatacheckEntry(r,[a],"DUPLICATE_LABEL")) - else: - all_route_labels.add(a) - # out-of-bounds coords if w.lat > 90 or w.lat < -90 or w.lng > 180 or w.lng < -180: datacheckerrors.append(DatacheckEntry(r,[w.label],'OUT_OF_BOUNDS', From 655bd7542425f0303543e5ec608750bcb44ae626 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Sun, 24 May 2020 11:26:39 -0400 Subject: [PATCH 09/34] 2 region-splitting bugfixes * write UTF-8 byte order mark to split-region .list files if present * write .list lines triggering "duplicate label" or "Equivalent waypoint labels mark zero distance traveled in line: " user log messages to split-region .list files --- .../cplusplus/classes/TravelerList/TravelerList.cpp | 5 ++++- .../TravelerList/mark_chopped_route_segments.cpp | 11 +++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp b/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp index b92dc698..6f5ff660 100644 --- a/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp +++ b/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp @@ -51,7 +51,10 @@ TravelerList::TravelerList(std::string travname, ErrorList *el, Arguments *args) } lines.push_back(listdata+listdatasize+1); // add a dummy "past-the-end" element to make lines[l+1]-2 work // strip UTF-8 byte order mark if present - if (!strncmp(lines[0], "\xEF\xBB\xBF", 3)) lines[0] += 3; + if (!strncmp(lines[0], "\xEF\xBB\xBF", 3)) + { lines[0] += 3; + splist << "\xEF\xBB\xBF"; + } for (unsigned int l = 0; l < lines.size()-1; l++) { std::string orig_line(lines[l]); diff --git a/siteupdate/cplusplus/classes/TravelerList/mark_chopped_route_segments.cpp b/siteupdate/cplusplus/classes/TravelerList/mark_chopped_route_segments.cpp index 0ebed9a1..ad6a2d00 100644 --- a/siteupdate/cplusplus/classes/TravelerList/mark_chopped_route_segments.cpp +++ b/siteupdate/cplusplus/classes/TravelerList/mark_chopped_route_segments.cpp @@ -35,14 +35,12 @@ while (*fields[2] == '*' || *fields[2] == '+') fields[2]++; while (*fields[3] == '*' || *fields[3] == '+') fields[3]++; upper(fields[2]); upper(fields[3]); - // look for point indices for labels, first in pri_label_hash std::unordered_map::iterator lit1 = r->pri_label_hash.find(fields[2]); std::unordered_map::iterator lit2 = r->pri_label_hash.find(fields[3]); // and then if not found, in alt_label_hash if (lit1 == r->pri_label_hash.end()) lit1 = r->alt_label_hash.find(fields[2]); if (lit2 == r->pri_label_hash.end()) lit2 = r->alt_label_hash.find(fields[3]); - // if we did not find matches for both labels... if (lit1 == r->alt_label_hash.end() || lit2 == r->alt_label_hash.end()) { bool invalid_char = 0; @@ -78,10 +76,15 @@ if (r->duplicate_labels.find(fields[3]) != r->duplicate_labels.end()) << ". Unable to parse line: " << trim_line << '\n'; duplicate = 1; } -if (duplicate) continue; +if (duplicate) +{ splist << orig_line << endlines[l]; + continue; +} // if both labels reference the same waypoint... if (lit1->second == lit2->second) - log << "Equivalent waypoint labels mark zero distance traveled in line: " << trim_line << '\n'; +{ log << "Equivalent waypoint labels mark zero distance traveled in line: " << trim_line << '\n'; + splist << orig_line << endlines[l]; +} // otherwise both labels are valid; mark in use & proceed else { r->system->lniu_mtx.lock(); r->system->listnamesinuse.insert(lookup); From 09e686417c033f4c0f2ebdb487bace7bae9d61d4 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Tue, 26 May 2020 22:00:09 -0400 Subject: [PATCH 10/34] trim whitespace from datacheckfps.csv lines --- siteupdate/cplusplus/siteupdate.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index 8b7c8ce4..2155906d 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -906,7 +906,12 @@ int main(int argc, char *argv[]) "NONTERMINAL_UNDERSCORE" }); while (getline(file, line)) - { if (line.back() == 0x0D) line.erase(line.end()-1); // trim DOS newlines + { // trim DOS newlines & trailing whitespace + while (line.back() == 0x0D || line.back() == ' ' || line.back() == '\t') + line.pop_back(); + // trim leading whitespace + while (line[0] == ' ' || line[0] == '\t') + line = line.substr(1); if (line.empty()) continue; // parse system updates.csv line size_t NumFields = 6; From 73ed9a3a6cd2ca61ee96c74a464ce6aa842b78f9 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Sun, 31 May 2020 02:33:52 -0400 Subject: [PATCH 11/34] Feature: Multi-region list file entries --- .../classes/ConnectedRoute/ConnectedRoute.cpp | 4 +- .../classes/ConnectedRoute/ConnectedRoute.h | 2 +- siteupdate/cplusplus/classes/Route/Route.cpp | 6 +- siteupdate/cplusplus/classes/Route/Route.h | 2 +- .../classes/TravelerList/TravelerList.cpp | 9 +- .../mark_chopped_route_segments.cpp | 17 +- .../mark_connected_route_segments.cpp | 159 ++++++++++++++++++ .../classes/TravelerList/splitregion.cpp | 62 +++++-- siteupdate/python-teresco/siteupdate.py | 139 ++++++++++++++- 9 files changed, 372 insertions(+), 28 deletions(-) create mode 100644 siteupdate/cplusplus/classes/TravelerList/mark_connected_route_segments.cpp diff --git a/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.cpp b/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.cpp index c043df6b..2e1d60e2 100644 --- a/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.cpp +++ b/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.cpp @@ -81,7 +81,7 @@ std::string ConnectedRoute::readable_name() return ans; } -std::string ConnectedRoute::list_lines(int pos, int len, std::string newline, size_t indent) +/*std::string ConnectedRoute::list_lines(int pos, int len, std::string newline, size_t indent) { // return .list file lines marking (len) consecutive // segments, starting at waypoint (pos) segments into route //std::cout << "\nDEBUG: list_lines for " << readable_name() << " (" << roots.size() << " connected root(s))" << std::endl; @@ -95,4 +95,4 @@ std::string ConnectedRoute::list_lines(int pos, int len, std::string newline, si // strip final newline while (lines.back() == '\n' || lines.back() == '\r') lines.pop_back(); return lines; -} +}//*/ diff --git a/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.h b/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.h index 9692f937..cf75dba6 100644 --- a/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.h +++ b/siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.h @@ -17,5 +17,5 @@ class ConnectedRoute std::string connected_rtes_line(); std::string csv_line(); std::string readable_name(); - std::string list_lines(int, int, std::string, size_t); + //std::string list_lines(int, int, std::string, size_t); }; diff --git a/siteupdate/cplusplus/classes/Route/Route.cpp b/siteupdate/cplusplus/classes/Route/Route.cpp index 1638cd3b..f11e86fa 100644 --- a/siteupdate/cplusplus/classes/Route/Route.cpp +++ b/siteupdate/cplusplus/classes/Route/Route.cpp @@ -160,16 +160,16 @@ double Route::clinched_by_traveler(TravelerList *t) return miles; } -std::string Route::list_line(int beg, int end) +/*std::string Route::list_line(int beg, int end) { /* Return a .list file line from (beg) to (end), these being indices to the point_list vector. These values can be "out-of-bounds" when getting lines for connected routes. If so, truncate or return "". */ - if (beg >= int(point_list.size()) || end <= 0) return ""; +/* if (beg >= int(point_list.size()) || end <= 0) return ""; if (end >= int(point_list.size())) end = point_list.size()-1; if (beg < 0) beg = 0; return readable_name() + " " + point_list[beg]->label + " " + point_list[end]->label; -} +}//*/ void Route::write_nmp_merged(std::string filename) { mkdir(filename.data(), 0777); diff --git a/siteupdate/cplusplus/classes/Route/Route.h b/siteupdate/cplusplus/classes/Route/Route.h index eaec3919..82f6ed6d 100644 --- a/siteupdate/cplusplus/classes/Route/Route.h +++ b/siteupdate/cplusplus/classes/Route/Route.h @@ -73,7 +73,7 @@ class Route std::string list_entry_name(); std::string name_no_abbrev(); double clinched_by_traveler(TravelerList *); - std::string list_line(int, int); + //std::string list_line(int, int); void write_nmp_merged(std::string); inline void store_traveled_segments(TravelerList*, unsigned int, unsigned int); }; diff --git a/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp b/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp index 6f5ff660..b3b10e6e 100644 --- a/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp +++ b/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp @@ -61,7 +61,7 @@ TravelerList::TravelerList(std::string travname, ErrorList *el, Arguments *args) // strip whitespace while (lines[l][0] == ' ' || lines[l][0] == '\t') lines[l]++; char * endchar = lines[l+1]-2; // -2 skips over the 0 inserted while separating listdata into lines - while (*endchar == 0) endchar--; // skip back more for CRLF cases, and lines followed by blank lines + while (*endchar == 0 && endchar > lines[l]) endchar--; // skip back more for CRLF cases, and lines followed by blank lines while (*endchar == ' ' || *endchar == '\t') { *endchar = 0; endchar--; @@ -84,7 +84,12 @@ TravelerList::TravelerList(std::string travname, ErrorList *el, Arguments *args) { #include "mark_chopped_route_segments.cpp" } - else { log << "Incorrect format line: " << trim_line << '\n'; + else if (fields.size() == 6) + { + #include "mark_connected_route_segments.cpp" + } + else { log << "Incorrect format line (4 or 6 fields expected, found " + << fields.size() << "): " << trim_line << '\n'; splist << orig_line << endlines[l]; } } diff --git a/siteupdate/cplusplus/classes/TravelerList/mark_chopped_route_segments.cpp b/siteupdate/cplusplus/classes/TravelerList/mark_chopped_route_segments.cpp index ad6a2d00..2be83dc9 100644 --- a/siteupdate/cplusplus/classes/TravelerList/mark_chopped_route_segments.cpp +++ b/siteupdate/cplusplus/classes/TravelerList/mark_chopped_route_segments.cpp @@ -19,7 +19,7 @@ if (rit == Route::pri_list_hash.end()) continue; } else log << "Note: deprecated route name " << fields[1] - << " -> canonical name " << rit->second->list_entry_name() << " in line " << trim_line << '\n'; + << " -> canonical name " << rit->second->list_entry_name() << " in line: " << trim_line << '\n'; } Route* r = rit->second; if (r->system->devel()) @@ -102,13 +102,24 @@ else { r->system->lniu_mtx.lock(); r->ual_mtx.unlock(); list_entries++; - if (lit1->second < lit2->second) + bool reverse = 0; + if (lit1->second <= lit2->second) { index1 = lit1->second; index2 = lit2->second; } else { index1 = lit2->second; index2 = lit1->second; + reverse = 1; } r->store_traveled_segments(this, index1, index2); - #include "splitregion.cpp" + // new .list lines for region split-ups + if (args->splitregion == r->region->code) + { + #define r1 r + #define r2 r + #include "splitregion.cpp" + #undef r1 + #undef r2 + } + else splist << orig_line << endlines[l]; } diff --git a/siteupdate/cplusplus/classes/TravelerList/mark_connected_route_segments.cpp b/siteupdate/cplusplus/classes/TravelerList/mark_connected_route_segments.cpp new file mode 100644 index 00000000..2ed64ef6 --- /dev/null +++ b/siteupdate/cplusplus/classes/TravelerList/mark_connected_route_segments.cpp @@ -0,0 +1,159 @@ +std::string lookup1 = upper(std::string(fields[0]) + ' ' + fields[1]); // leave fields intact for potential AltRouteName note +std::string lookup2 = upper(std::string(fields[3]) + ' ' + fields[4]); // leave fields intact for potential AltRouteName note +// look for region/route combos, first in pri_list_hash +std::unordered_map::iterator rit1 = Route::pri_list_hash.find(lookup1); +std::unordered_map::iterator rit2 = Route::pri_list_hash.find(lookup2); +// and then if not found, in alt_list_hash +if (rit1 == Route::pri_list_hash.end()) +{ rit1 = Route::alt_list_hash.find(lookup1); + if (rit1 != Route::alt_list_hash.end()) + log << "Note: deprecated route name \"" << fields[0] << ' ' << fields[1] + << "\" -> canonical name \"" << rit1->second->readable_name() << "\" in line: " << trim_line << '\n'; +} +if (rit2 == Route::pri_list_hash.end()) +{ rit2 = Route::alt_list_hash.find(lookup2); + if (rit2 != Route::alt_list_hash.end()) + log << "Note: deprecated route name \"" << fields[3] << ' ' << fields[4] + << "\" -> canonical name \"" << rit2->second->readable_name() << "\" in line: " << trim_line << '\n'; +} +if (rit1 == Route::alt_list_hash.end() || rit2 == Route::alt_list_hash.end()) +{ bool invalid_char = 0; + for (char& c : trim_line) + if (iscntrl(c)) + { c = '?'; + invalid_char = 1; + } + for (char& c : lookup1) if (iscntrl(c)) c = '?'; + for (char& c : lookup2) if (iscntrl(c)) c = '?'; + if (rit1 == rit2) + log << "Unknown region/highway combos " << lookup1 << " and " << lookup2 << " in line: " << trim_line; + else { log << "Unknown region/highway combo "; + log << (rit1 == Route::alt_list_hash.end() ? lookup1 : lookup2); + log << " in line: " << trim_line; + } + if (invalid_char) log << " [contains invalid character(s)]"; + log << '\n'; + splist << orig_line << endlines[l]; + continue; +} +Route* r1 = rit1->second; +Route* r2 = rit2->second; +if (r1->con_route != r2->con_route) +{ log << lookup1 << " and " << lookup2 << " not in same connected route in line: " << trim_line << '\n'; + splist << orig_line << endlines[l]; + continue; +} +if (r1->system->devel()) +{ log << "Ignoring line matching highway in system in development: " << trim_line << '\n'; + splist << orig_line << endlines[l]; + continue; +} +// r1 and r2 are route matches, and we need to find +// waypoint indices, ignoring case and leading +// '+' or '*' when matching +while (*fields[2] == '*' || *fields[2] == '+') fields[2]++; +while (*fields[5] == '*' || *fields[5] == '+') fields[5]++; +upper(fields[2]); +upper(fields[5]); +// look for point indices for labels, first in pri_label_hash +std::unordered_map::iterator lit1 = r1->pri_label_hash.find(fields[2]); +std::unordered_map::iterator lit2 = r2->pri_label_hash.find(fields[5]); +// and then if not found, in alt_label_hash +if (lit1 == r1->pri_label_hash.end()) lit1 = r1->alt_label_hash.find(fields[2]); +if (lit2 == r2->pri_label_hash.end()) lit2 = r2->alt_label_hash.find(fields[5]); +// if we did not find matches for both labels... +if (lit1 == r1->alt_label_hash.end() || lit2 == r2->alt_label_hash.end()) +{ bool invalid_char = 0; + for (char& c : trim_line) + if (iscntrl(c)) + { c = '?'; + invalid_char = 1; + } + for (char* c = fields[2]; *c; c++) if (iscntrl(*c)) *c = '?'; + for (char* c = fields[5]; *c; c++) if (iscntrl(*c)) *c = '?'; + if (lit1 == r1->alt_label_hash.end() && lit2 == r2->alt_label_hash.end()) + log << "Waypoint labels " << fields[2] << " and " << fields[5] << " not found in line: " << trim_line; + else { log << "Waypoint "; + if (lit1 == r1->alt_label_hash.end()) + log << lookup1 << ' ' << fields[2]; + else log << lookup2 << ' ' << fields[5]; + log << " not found in line: " << trim_line; + } + if (invalid_char) log << " [contains invalid character(s)]"; + log << '\n'; + splist << orig_line << endlines[l]; + continue; +} +// are either of the labels used duplicates? +char duplicate = 0; +if (r1->duplicate_labels.find(fields[2]) != r1->duplicate_labels.end()) +{ log << r1->region->code << ": duplicate label " << fields[2] << " in " << r1->root + << ". Please report this error in the TravelMapping forum" + << ". Unable to parse line: " << trim_line << '\n'; + duplicate = 1; +} +if (r2->duplicate_labels.find(fields[5]) != r2->duplicate_labels.end()) +{ log << r2->region->code << ": duplicate label " << fields[5] << " in " << r2->root + << ". Please report this error in the TravelMapping forum" + << ". Unable to parse line: " << trim_line << '\n'; + duplicate = 1; +} +if (duplicate) +{ splist << orig_line << endlines[l]; + continue; +} +bool reverse = 0; +unsigned int index1 = lit1->second; +unsigned int index2 = lit2->second; +// if both region/route combos point to the same chopped route... +if (r1 == r2) + { // if both labels reference the same waypoint... + if (index1 == index2) + { log << "Equivalent waypoint labels mark zero distance traveled in line: " << trim_line << '\n'; + splist << orig_line << endlines[l]; + continue; + } + if (index1 <= index2) + r1->store_traveled_segments(this, index1, index2); + else r1->store_traveled_segments(this, index2, index1); + } +else { if (r1->rootOrder > r2->rootOrder) + { std::swap(r1, r2); + index1 = lit2->second; + index2 = lit1->second; + reverse = 1; + } + // mark the beginning chopped route from index1 to its end + r1->store_traveled_segments(this, index1, r1->segment_list.size()); + // mark the ending chopped route from its beginning to index2 + r2->store_traveled_segments(this, 0, index2); + // mark any intermediate chopped routes in their entirety. + for (size_t r = r1->rootOrder+1; r < r2->rootOrder; r++) + r1->con_route->roots[r]->store_traveled_segments(this, 0, r1->con_route->roots[r]->segment_list.size()); + } +// both labels are valid; mark in use & proceed +r1->system->lniu_mtx.lock(); +r1->system->listnamesinuse.insert(lookup1); +r1->system->lniu_mtx.unlock(); +r1->system->uarn_mtx.lock(); +r1->system->unusedaltroutenames.erase(lookup1); +r1->system->uarn_mtx.unlock(); +r1->liu_mtx.lock(); +r1->labels_in_use.insert(fields[2]); +r1->liu_mtx.unlock(); +r2->system->lniu_mtx.lock(); +r2->system->listnamesinuse.insert(lookup2); +r2->system->lniu_mtx.unlock(); +r2->system->uarn_mtx.lock(); +r2->system->unusedaltroutenames.erase(lookup2); +r2->system->uarn_mtx.unlock(); +r2->liu_mtx.lock(); +r2->labels_in_use.insert(fields[5]); +r2->liu_mtx.unlock(); +list_entries++; +// new .list lines for region split-ups +if (args->splitregion == r1->region->code || args->splitregion == r2->region->code) +{ + #include "splitregion.cpp" +} +else splist << orig_line << endlines[l]; diff --git a/siteupdate/cplusplus/classes/TravelerList/splitregion.cpp b/siteupdate/cplusplus/classes/TravelerList/splitregion.cpp index a3f967bd..56f1dc70 100644 --- a/siteupdate/cplusplus/classes/TravelerList/splitregion.cpp +++ b/siteupdate/cplusplus/classes/TravelerList/splitregion.cpp @@ -1,24 +1,66 @@ -// new .list lines for region split-ups -if (args->splitregion == upper(fields[0])) -{ // first, comment out original line - splist << "##### " << orig_line << newline; - HighwaySegment *orig_hs = r->segment_list[index1]; +Waypoint *w1, *w2; +// comment out original line and indent new line below +splist << "##### " << orig_line << newline << " "; + +// 1st waypoint +if (args->splitregion != r1->region->code) + // if not in splitregion, just use canonical .list name and waypoint label + w1 = r1->point_list[index1]; +else { HighwaySegment *orig_hs = r1->segment_list[index1 == r1->segment_list.size() ? index1-1 : index1]; HighwaySegment *new_hs = 0; if (!orig_hs->concurrent) std::cout << "ERROR: " << orig_hs->str() << " not concurrent" << std::endl; else { size_t count = 0; // find concurrent segment with same name in different region for (HighwaySegment *cs : *(orig_hs->concurrent)) - if (cs->route->name_no_abbrev() == r->name_no_abbrev() && cs->route->region != r->region) + if (cs->route->name_no_abbrev() == r1->name_no_abbrev() && cs->route->region != r1->region) { count++; new_hs = cs; } if (!new_hs) std::cout << "ERROR: concurrent segment not found for " << orig_hs->str() << std::endl; else { if (count > 1) std::cout << "DEBUG: multiple matches found for " << orig_hs->str() << std::endl; - // get lines from associated connected route. // assumption: each chopped route in old full region corresponds 1:1 to a connected route in new chopped regions - splist << new_hs->route->con_route->list_lines(index1, index2 - index1, newline, 2) << endlines[l]; + // index within old chopped route = index within new connected route. + // convert to index within new chopped route + for (long int r = new_hs->route->rootOrder; r > 0; r--) + index1 -= new_hs->route->con_route->roots[r-1]->segment_list.size(); + w1 = new_hs->route->point_list[index1]; } } -} -else splist << orig_line << endlines[l]; + } + +// 2nd waypoint +if (args->splitregion != r2->region->code) + // if not in splitregion, just use canonical .list name and waypoint label + w2 = r2->point_list[index2]; +else { HighwaySegment *orig_hs = r2->segment_list[index2 ? index2-1 : 0]; + HighwaySegment *new_hs = 0; + if (!orig_hs->concurrent) + std::cout << "ERROR: " << orig_hs->str() << " not concurrent" << std::endl; + else { size_t count = 0; + // find concurrent segment with same name in different region + for (HighwaySegment *cs : *(orig_hs->concurrent)) + if (cs->route->name_no_abbrev() == r2->name_no_abbrev() && cs->route->region != r2->region) + { count++; + new_hs = cs; + } + if (!new_hs) std::cout << "ERROR: concurrent segment not found for " << orig_hs->str() << std::endl; + else { if (count > 1) std::cout << "DEBUG: multiple matches found for " << orig_hs->str() << std::endl; + // assumption: each chopped route in old full region corresponds 1:1 to a connected route in new chopped regions + // index within old chopped route = index within new connected route. + // convert to index within new chopped route + for (long int r = new_hs->route->rootOrder; r > 0; r--) + index2 -= new_hs->route->con_route->roots[r-1]->segment_list.size(); + w2 = new_hs->route->point_list[index2]; + } + } + } + +// write new line to file +if (w1->route == w2->route) // 4 fields possible + if (index1 > index2 || reverse) + splist << w2->route->readable_name() << ' ' << w2->label << ' ' << w1->label << endlines[l]; + else splist << w1->route->readable_name() << ' ' << w1->label << ' ' << w2->label << endlines[l]; +else if (reverse) + splist << w2->route->readable_name() << ' ' << w2->label << ' ' << w1->route->readable_name() << ' ' << w1->label << endlines[l]; + else splist << w1->route->readable_name() << ' ' << w1->label << ' ' << w2->route->readable_name() << ' ' << w2->label << endlines[l]; diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 66472c33..ef61f481 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Travel Mapping Project, Jim Teresco, 2015-2018 +# Travel Mapping Project, Jim Teresco, 2015-2020 """Python code to read .csv and .wpt files and prepare for adding to the Travel Mapping Project database. @@ -1284,7 +1284,7 @@ def __init__(self,travelername,el,path="../../../UserData/list_files"): continue fields = re.split('[ \t]+',line) # truncate inline comments from fields list - for i in range(min(len(fields), 5)): + for i in range(min(len(fields), 7)): if fields[i][0] == '#': fields = fields[:i] break @@ -1305,7 +1305,7 @@ def __init__(self,travelername,el,path="../../../UserData/list_files"): self.log_entries[-1] += " [contains invalid character(s)]" continue else: - self.log_entries.append("Note: deprecated route name " + fields[1] + " -> canonical name " + r.list_entry_name() + " in line " + line) + self.log_entries.append("Note: deprecated route name " + fields[1] + " -> canonical name " + r.list_entry_name() + " in line: " + line) if r.system.devel(): self.log_entries.append("Ignoring line matching highway in system in development: " + line) continue @@ -1378,12 +1378,139 @@ def __init__(self,travelername,el,path="../../../UserData/list_files"): if index1 > index2: (index1, index2) = (index2, index1) r.store_traveled_segments(self, index1, index2) + elif len(fields) == 6: + lookup1 = fields[0].upper() + ' ' + fields[1].upper() + lookup2 = fields[3].upper() + ' ' + fields[4].upper() + # look for region/route combos, first in pri_list_hash + # and then if not found, in alt_list_hash + both_lookups_found = True + try: + r1 = Route.pri_list_hash[lookup1] + except KeyError: + try: + r1 = Route.alt_list_hash[lookup1] + except KeyError: + r1 = None + else: + self.log_entries.append("Note: deprecated route name \"" + fields[0] + ' ' + fields[1] \ + + "\" -> canonical name \"" + r1.readable_name() + "\" in line: " + line) + try: + r2 = Route.pri_list_hash[lookup2] + except KeyError: + try: + r2 = Route.alt_list_hash[lookup2] + except KeyError: + r2 = None + else: + self.log_entries.append("Note: deprecated route name \"" + fields[3] + ' ' + fields[4] \ + + "\" -> canonical name \"" + r2.readable_name() + "\" in line: " + line) + if r1 is None or r2 is None: + (lookup1, invchar) = no_control_chars(lookup1) + (lookup2, invchar) = no_control_chars(lookup2) + (line, invchar) = no_control_chars(line) + if r1 == r2: + self.log_entries.append("Unknown region/highway combos " + lookup1 + " and " + lookup2 + " in line: " + line) + else: + log_entry = "Unknown region/highway combo " + log_entry += lookup1 if r1 is None else lookup2 + log_entry += " in line: " + line + self.log_entries.append(log_entry) + if invchar: + self.log_entries[-1] += " [contains invalid character(s)]" + continue + if r1.con_route != r2.con_route: + self.log_entries.append(lookup1 + " and " + lookup2 + " not in same connected route in line: " + line) + continue + if r1.system.devel(): + self.log_entries.append("Ignoring line matching highway in system in development: " + line) + continue + # r1 and r2 are route matches, and we need to find + # waypoint indices, ignoring case and leading + # '+' or '*' when matching + list_label_1 = fields[2].lstrip("+*").upper() + list_label_2 = fields[5].lstrip("+*").upper() + # look for point indices for labels, first in pri_label_hash + # and then if not found, in alt_label_hash + try: + index1 = r1.pri_label_hash[list_label_1] + except KeyError: + try: + index1 = r1.alt_label_hash[list_label_1] + except KeyError: + index1 = None + try: + index2 = r2.pri_label_hash[list_label_2] + except KeyError: + try: + index2 = r2.alt_label_hash[list_label_2] + except KeyError: + index2 = None + # if we did not find matches for both labels... + if index1 is None or index2 is None: + (list_label_1, invchar) = no_control_chars(list_label_1) + (list_label_2, invchar) = no_control_chars(list_label_2) + (line, invchar) = no_control_chars(line) + if index1 is None and index2 is None: + self.log_entries.append("Waypoint labels " + list_label_1 + " and " + list_label_2 + " not found in line: " + line) + else: + log_entry = "Waypoint " + if index1 is None: + log_entry += lookup1 + ' ' + list_label_1 + else: + log_entry += lookup2 + ' ' + list_label_2 + log_entry += " not found in line: " + line + self.log_entries.append(log_entry) + if invchar: + self.log_entries[-1] += " [contains invalid character(s)]" + continue + # are either of the labels used duplicates? + duplicate = False + if list_label_1 in r1.duplicate_labels: + self.log_entries.append(r1.region + ": duplicate label " + list_label_1 + " in " + r1.root \ + + ". Please report this error in the TravelMapping forum. Unable to parse line: " + line) + duplicate = True + if list_label_2 in r2.duplicate_labels: + self.log_entries.append(r2.region + ": duplicate label " + list_label_2 + " in " + r2.root \ + + ". Please report this error in the TravelMapping forum. Unable to parse line: " + line) + duplicate = True + if duplicate: + continue + # if both region/route combos point to the same chopped route... + if r1 == r2: + # if both labels reference the same waypoint... + if index1 == index2: + self.log_entries.append("Equivalent waypoint labels mark zero distance traveled in line: " + line) + continue + if index1 <= index2: + r1.store_traveled_segments(self, index1, index2) + else: + r1.store_traveled_segments(self, index2, index1) + else: + if r1.rootOrder > r2.rootOrder: + (r1, r2) = (r2, r1) + (index1, index2) = (index2, index1) + # mark the beginning chopped route from index1 to its end + r1.store_traveled_segments(self, index1, len(r1.segment_list)) + # mark the ending chopped route from its beginning to index2 + r2.store_traveled_segments(self, 0, index2) + # mark any intermediate chopped routes in their entirety. + for r in range(r1.rootOrder+1, r2.rootOrder): + r1.con_route.roots[r].store_traveled_segments(self, 0, len(r1.con_route.roots[r].segment_list)) + # both labels are valid; mark in use & proceed + r1.system.listnamesinuse.add(lookup1) + r1.system.unusedaltroutenames.discard(lookup1) + r1.labels_in_use.add(list_label_1) + r2.system.listnamesinuse.add(lookup2) + r2.system.unusedaltroutenames.discard(lookup2) + r2.labels_in_use.add(list_label_2) + list_entries += 1 else: - self.log_entries.append("Incorrect format line: " + line) + self.log_entries.append("Incorrect format line (4 or 6 fields expected, found " + \ + str(len(fields)) +"): " + line) self.log_entries.append("Processed " + str(list_entries) + \ - " good lines marking " +str(len(self.clinched_segments)) + \ - " segments traveled.") + " good lines marking " +str(len(self.clinched_segments)) + \ + " segments traveled.") # additional setup for later stats processing # a place to track this user's total mileage per region, # but only active+preview and active only (since devel From c15f3c2d660e993c5758d05ea71d0369667c962b Mon Sep 17 00:00:00 2001 From: eric bryant Date: Sun, 31 May 2020 20:35:57 -0400 Subject: [PATCH 12/34] mutex fixes --- siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp | 5 ++++- siteupdate/cplusplus/classes/TravelerList/TravelerList.h | 4 +++- siteupdate/cplusplus/siteupdate.cpp | 4 ++-- siteupdate/cplusplus/threads/ConcAugThread.cpp | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp b/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp index b3b10e6e..6598965e 100644 --- a/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp +++ b/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp @@ -13,7 +13,10 @@ TravelerList::TravelerList(std::string travname, ErrorList *el, Arguments *args) std::ofstream splist; if (args->splitregionpath != "") splist.open(args->splitregionpath+"/list_files/"+travname); time_t StartTime = time(0); - log << "Log file created at: " << ctime(&StartTime); + log << "Log file created at: "; + alltrav_mtx.lock(); + log << ctime(&StartTime); + alltrav_mtx.unlock(); std::vector lines; std::vector endlines; std::ifstream file(args->userlistfilepath+"/"+travname); diff --git a/siteupdate/cplusplus/classes/TravelerList/TravelerList.h b/siteupdate/cplusplus/classes/TravelerList/TravelerList.h index 9bbceb10..7710dee9 100644 --- a/siteupdate/cplusplus/classes/TravelerList/TravelerList.h +++ b/siteupdate/cplusplus/classes/TravelerList/TravelerList.h @@ -27,7 +27,9 @@ class TravelerList unsigned int active_systems_clinched; unsigned int preview_systems_traveled; unsigned int preview_systems_clinched; - static std::mutex alltrav_mtx; // for locking the traveler_lists list when reading .lists from disk + // for locking the traveler_lists list when reading .lists from disk + // and avoiding data races when creating userlog timestamps + static std::mutex alltrav_mtx; TravelerList(std::string, ErrorList *, Arguments *); double active_only_miles(); diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index 2155906d..79bc8322 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -97,7 +97,7 @@ using namespace std; int main(int argc, char *argv[]) { ifstream file; string line; - mutex list_mtx, log_mtx; + mutex list_mtx; time_t timestamp; // start a timer for including elapsed time reports in messages @@ -630,7 +630,7 @@ int main(int argc, char *argv[]) list::iterator tl_it = traveler_lists.begin(); for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(ConcAugThread, t, &traveler_lists, &tl_it, &list_mtx, &log_mtx, augment_lists+t); + thr[t] = new thread(ConcAugThread, t, &traveler_lists, &tl_it, &list_mtx, augment_lists+t); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); cout << "!\n" << et.et() << "Writing to concurrencies.log." << endl; diff --git a/siteupdate/cplusplus/threads/ConcAugThread.cpp b/siteupdate/cplusplus/threads/ConcAugThread.cpp index 9a848185..f7c3b5ef 100644 --- a/siteupdate/cplusplus/threads/ConcAugThread.cpp +++ b/siteupdate/cplusplus/threads/ConcAugThread.cpp @@ -1,5 +1,5 @@ void ConcAugThread(unsigned int id, std::list *travlists, std::list::iterator *it, - std::mutex *tl_mtx, std::mutex *log_mtx, std::list* augment_list) + std::mutex *tl_mtx, std::list* augment_list) { //printf("Starting ConcAugThread %02i\n", id); fflush(stdout); while (*it != travlists->end()) { tl_mtx->lock(); From e40e80b3a44c92c330dd44c3412f771fa1e7068a Mon Sep 17 00:00:00 2001 From: eric bryant Date: Tue, 2 Jun 2020 00:01:17 -0400 Subject: [PATCH 13/34] LABEL_INVALID_CHAR fix --- .../cplusplus/classes/Route/read_wpt.cpp | 3 +-- .../cplusplus/classes/Waypoint/Waypoint.cpp | 22 ++++++++++--------- .../cplusplus/classes/Waypoint/Waypoint.h | 2 +- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/siteupdate/cplusplus/classes/Route/read_wpt.cpp b/siteupdate/cplusplus/classes/Route/read_wpt.cpp index f8d3db14..4af30af2 100644 --- a/siteupdate/cplusplus/classes/Route/read_wpt.cpp +++ b/siteupdate/cplusplus/classes/Route/read_wpt.cpp @@ -76,8 +76,7 @@ void Route::read_wpt w->label_slashes(datacheckerrors, slash); w->underscore_datachecks(datacheckerrors, slash); w->label_parens(datacheckerrors); - w->label_invalid_char(datacheckerrors, w->label); - for (std::string &a : w->alt_labels) w->label_invalid_char(datacheckerrors, a); + w->label_invalid_char(datacheckerrors); w->label_invalid_ends(datacheckerrors); w->bus_with_i(datacheckerrors); w->label_looks_hidden(datacheckerrors); diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp index 3eea81ef..812f0df1 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp @@ -398,17 +398,19 @@ inline void Waypoint::label_looks_hidden(DatacheckEntryList *datacheckerrors) datacheckerrors->add(route, label, "", "", "LABEL_LOOKS_HIDDEN", ""); } -inline void Waypoint::label_invalid_char(DatacheckEntryList *datacheckerrors, std::string &lbl) +inline void Waypoint::label_invalid_char(DatacheckEntryList *datacheckerrors) { // look for labels with invalid characters - for (const char *c = lbl.data(); *c; c++) - { if (*c < 40) { datacheckerrors->add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); return; } - if (*c == 44) { datacheckerrors->add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); return; } - if (*c > 57 && *c < 65) { datacheckerrors->add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); return; } - if (*c > 90 && *c < 95) { datacheckerrors->add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); return; } - if (*c == 96) { datacheckerrors->add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); return; } - if (*c > 122) { datacheckerrors->add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); return; } - } - if (strpbrk(lbl.data()+1, "+*")) datacheckerrors->add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); + for (const char *c = label.data(); *c; c++) + if ((*c == 42 || *c == 43) && c > label.data() + || (*c < 40) || (*c == 44) || (*c > 57 && *c < 65) + || (*c == 96) || (*c > 122) || (*c > 90 && *c < 95)) + datacheckerrors->add(route, label, "", "", "LABEL_INVALID_CHAR", ""); + for (std::string& lbl : alt_labels) + for (const char *c = lbl.data(); *c; c++) + if (*c == '+' && c > lbl.data() || *c == '*' && (c > lbl.data()+1 || lbl[0] != '+') + || (*c < 40) || (*c == 44) || (*c > 57 && *c < 65) + || (*c == 96) || (*c > 122) || (*c > 90 && *c < 95)) + datacheckerrors->add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); } inline void Waypoint::label_invalid_ends(DatacheckEntryList *datacheckerrors) diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.h b/siteupdate/cplusplus/classes/Waypoint/Waypoint.h index 7e302c8c..9d7ddc2c 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.h +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.h @@ -46,7 +46,7 @@ class Waypoint inline void visible_distance(DatacheckEntryList *, char *, double &, Waypoint *&); inline void bus_with_i(DatacheckEntryList *); inline void label_looks_hidden(DatacheckEntryList *); - inline void label_invalid_char(DatacheckEntryList *, std::string &); + inline void label_invalid_char(DatacheckEntryList *); inline void label_invalid_ends(DatacheckEntryList *); inline void label_parens(DatacheckEntryList *); inline void label_slashes(DatacheckEntryList *, const char *); From 166286b802602e9cef0bbf0d4fff8eec88a6d954 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Wed, 10 Jun 2020 21:46:07 -0400 Subject: [PATCH 14/34] ConnectedRoute checks & fixes * 6-field .list entries cope with reversed point order within routes http://forum.travelmapping.net/index.php?topic=3648.msg19036#msg19036 http://forum.travelmapping.net/index.php?topic=3652.msg19105#msg19105 * Implement DISCONNECTED_ROUTE datacheck http://forum.travelmapping.net/index.php?topic=3652 --- siteupdate/cplusplus/classes/Route/Route.cpp | 9 +++++ siteupdate/cplusplus/classes/Route/Route.h | 3 ++ .../mark_connected_route_segments.cpp | 8 +++- siteupdate/cplusplus/siteupdate.cpp | 37 +++++++++++++++++-- siteupdate/python-teresco/siteupdate.py | 35 +++++++++++++++++- 5 files changed, 85 insertions(+), 7 deletions(-) diff --git a/siteupdate/cplusplus/classes/Route/Route.cpp b/siteupdate/cplusplus/classes/Route/Route.cpp index f11e86fa..609de387 100644 --- a/siteupdate/cplusplus/classes/Route/Route.cpp +++ b/siteupdate/cplusplus/classes/Route/Route.cpp @@ -8,6 +8,7 @@ Route::Route(std::string &line, HighwaySystem *sys, ErrorList &el, std::unordere mileage = 0; rootOrder = -1; // order within connected route region = 0; // if this stays 0, setup has failed due to bad .csv data + is_reversed = 0; // parse chopped routes csv line size_t NumFields = 8; @@ -216,3 +217,11 @@ inline void Route::store_traveled_segments(TravelerList* t, unsigned int beg, un t->clinched_segments.insert(hs); } } + +inline Waypoint* Route::con_beg() +{ return is_reversed ? point_list.back() : point_list.front(); +} + +inline Waypoint* Route::con_end() +{ return is_reversed ? point_list.front() : point_list.back(); +} diff --git a/siteupdate/cplusplus/classes/Route/Route.h b/siteupdate/cplusplus/classes/Route/Route.h index 82f6ed6d..e099d6db 100644 --- a/siteupdate/cplusplus/classes/Route/Route.h +++ b/siteupdate/cplusplus/classes/Route/Route.h @@ -60,6 +60,7 @@ class Route std::vector segment_list; double mileage; int rootOrder; + bool is_reversed; Route(std::string &, HighwaySystem *, ErrorList &, std::unordered_map &); @@ -76,4 +77,6 @@ class Route //std::string list_line(int, int); void write_nmp_merged(std::string); inline void store_traveled_segments(TravelerList*, unsigned int, unsigned int); + inline Waypoint* con_beg(); + inline Waypoint* con_end(); }; diff --git a/siteupdate/cplusplus/classes/TravelerList/mark_connected_route_segments.cpp b/siteupdate/cplusplus/classes/TravelerList/mark_connected_route_segments.cpp index 2ed64ef6..61f70421 100644 --- a/siteupdate/cplusplus/classes/TravelerList/mark_connected_route_segments.cpp +++ b/siteupdate/cplusplus/classes/TravelerList/mark_connected_route_segments.cpp @@ -124,9 +124,13 @@ else { if (r1->rootOrder > r2->rootOrder) reverse = 1; } // mark the beginning chopped route from index1 to its end - r1->store_traveled_segments(this, index1, r1->segment_list.size()); + if (r1->is_reversed) + r1->store_traveled_segments(this, 0, index1); + else r1->store_traveled_segments(this, index1, r1->segment_list.size()); // mark the ending chopped route from its beginning to index2 - r2->store_traveled_segments(this, 0, index2); + if (r2->is_reversed) + r2->store_traveled_segments(this, index2, r2->segment_list.size()); + else r2->store_traveled_segments(this, 0, index2); // mark any intermediate chopped routes in their entirety. for (size_t r = r1->rootOrder+1; r < r2->rootOrder; r++) r1->con_route->roots[r]->store_traveled_segments(this, 0, r1->con_route->roots[r]->segment_list.size()); diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index 79bc8322..06a89891 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -391,8 +391,39 @@ int main(int argc, char *argv[]) { // check for unconnected chopped routes if (!r.con_route) el.add_error(r.system->systemname + ".csv: root " + r.root + " not matched by any connected route root."); - unsigned int index = 0; - for (Waypoint* w : r.point_list) + + // check for mismatched route endpoints within connected routes + #define q r.con_route->roots[r.rootOrder-1] + if ( r.rootOrder > 0 && q->point_list.size() > 1 && !r.con_beg()->same_coords(q->con_end()) ) + { if ( q->con_beg()->same_coords(r.con_beg()) ) + { //std::cout << "DEBUG: marking only " << q->str() << " reversed" << std::endl; + //if (q->is_reversed) std::cout << "DEBUG: " << q->str() << " already reversed!" << std::endl; + q->is_reversed = 1; + } + else if ( q->con_end()->same_coords(r.con_end()) ) + { //std::cout << "DEBUG: marking only " << r.str() << " reversed" << std::endl; + //if (r.is_reversed) std::cout << "DEBUG: " << r.str() << " already reversed!" << std::endl; + r.is_reversed = 1; + } + else if ( q->con_beg()->same_coords(r.con_end()) ) + { //std::cout << "DEBUG: marking both " << q->str() << " and " << r.str() << " reversed" << std::endl; + //if (q->is_reversed) std::cout << "DEBUG: " << q->str() << " already reversed!" << std::endl; + //if (r.is_reversed) std::cout << "DEBUG: " << r.str() << " already reversed!" << std::endl; + q->is_reversed = 1; + r.is_reversed = 1; + } + else if ( !q->con_end()->same_coords(r.con_beg()) ) + { datacheckerrors->add(&r, r.con_beg()->label, "", "", + "DISCONNECTED_ROUTE", q->con_end()->root_at_label()); + datacheckerrors->add(q, q->con_end()->label, "", "", + "DISCONNECTED_ROUTE", r.con_beg()->root_at_label()); + } + } + #undef q + + // create label hashes and check for duplicates + #define w r.point_list[index] + for (unsigned int index = 0; index < r.point_list.size(); index++) { // ignore case and leading '+' or '*' std::string upper_label = upper(w->label); while (upper_label[0] == '+' || upper_label[0] == '*') @@ -422,8 +453,8 @@ int main(int argc, char *argv[]) r.duplicate_labels.insert(a); } } - index++; } + #undef w } // Create a list of TravelerList objects, one per person diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index ef61f481..c673d166 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -990,6 +990,7 @@ def __init__(self,line,system,el): self.con_route = None self.mileage = 0.0 self.rootOrder = -1 # order within connected route + self.is_reversed = False def __str__(self): """printable version of the object""" @@ -1095,6 +1096,12 @@ def store_traveled_segments(self, t, beg, end): hs.add_clinched_by(t) t.clinched_segments.add(hs) + def con_beg(self): + return self.point_list[-1] if self.is_reversed else self.point_list[0] + + def con_end(self): + return self.point_list[0] if self.is_reversed else self.point_list[-1] + class ConnectedRoute: """This class encapsulates a single 'connected route' as given by a single line of a _con.csv file @@ -1490,9 +1497,15 @@ def __init__(self,travelername,el,path="../../../UserData/list_files"): (r1, r2) = (r2, r1) (index1, index2) = (index2, index1) # mark the beginning chopped route from index1 to its end - r1.store_traveled_segments(self, index1, len(r1.segment_list)) + if r1.is_reversed: + r1.store_traveled_segments(self, 0, index1) + else: + r1.store_traveled_segments(self, index1, len(r1.segment_list)) # mark the ending chopped route from its beginning to index2 - r2.store_traveled_segments(self, 0, index2) + if r2.is_reversed: + r2.store_traveled_segments(self, index2, len(r2.segment_list)) + else: + r2.store_traveled_segments(self, 0, index2) # mark any intermediate chopped routes in their entirety. for r in range(r1.rootOrder+1, r2.rootOrder): r1.con_route.roots[r].store_traveled_segments(self, 0, len(r1.con_route.roots[r].segment_list)) @@ -2873,6 +2886,24 @@ def run(self): # check for unconnected chopped routes if r.con_route is None: el.add_error(r.system.systemname + ".csv: root " + r.root + " not matched by any connected route root.") + + # check for mismatched route endpoints within connected routes + q = r.con_route.roots[r.rootOrder-1] + if r.rootOrder > 0 and len(q.point_list) > 1 and not r.con_beg().same_coords(q.con_end()): + if q.con_beg().same_coords(r.con_beg()): + q.is_reversed = True + elif q.con_end().same_coords(r.con_end()): + r.is_reversed = True + elif q.con_beg().same_coords(r.con_end()): + q.is_reversed = True + r.is_reversed = True + elif not q.con_end().same_coords(r.con_beg()): + datacheckerrors.append(DatacheckEntry(r, [r.con_beg().label], + "DISCONNECTED_ROUTE", q.root + '@' + q.con_end().label)) + datacheckerrors.append(DatacheckEntry(q, [q.con_end().label], + "DISCONNECTED_ROUTE", r.root + '@' + r.con_beg().label)) + + # create label hashes and check for duplicates index = 0 for w in r.point_list: # ignore case and leading '+' or '*' From b2b7e13c60547c19903a0f77f8719db903c8106e Mon Sep 17 00:00:00 2001 From: eric bryant Date: Wed, 10 Jun 2020 21:59:55 -0400 Subject: [PATCH 15/34] DISCONNECTED_ROUTE comment & always_error entry * add to list of all error codes with description of add'l info * add to datacheck_always _error --- siteupdate/cplusplus/classes/DatacheckEntry.cpp | 1 + siteupdate/cplusplus/siteupdate.cpp | 4 ++-- siteupdate/python-teresco/siteupdate.py | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/siteupdate/cplusplus/classes/DatacheckEntry.cpp b/siteupdate/cplusplus/classes/DatacheckEntry.cpp index 8aa0ec0d..99f7cd61 100644 --- a/siteupdate/cplusplus/classes/DatacheckEntry.cpp +++ b/siteupdate/cplusplus/classes/DatacheckEntry.cpp @@ -12,6 +12,7 @@ class DatacheckEntry -----------------------+-------------------------------------------- BAD_ANGLE | BUS_WITH_I | + DISCONNECTED_ROUTE | adjacent root's expected connection point DUPLICATE_COORDS | coordinate pair DUPLICATE_LABEL | HIDDEN_JUNCTION | number of incident edges in TM master graph diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index 06a89891..91612622 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -929,8 +929,8 @@ int main(int argc, char *argv[]) getline(file, line); // ignore header line list> datacheckfps; unordered_set datacheck_always_error - ({ "BAD_ANGLE", "DUPLICATE_LABEL", "HIDDEN_TERMINUS", - "INVALID_FINAL_CHAR", "INVALID_FIRST_CHAR", + ({ "BAD_ANGLE", "DISCONNECTED_ROUTE", "DUPLICATE_LABEL", + "HIDDEN_TERMINUS", "INVALID_FINAL_CHAR", "INVALID_FIRST_CHAR", "LABEL_INVALID_CHAR", "LABEL_PARENS", "LABEL_SLASHES", "LABEL_TOO_LONG", "LABEL_UNDERSCORES", "LONG_UNDERSCORE", "MALFORMED_LAT", "MALFORMED_LON", "MALFORMED_URL", diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index c673d166..c6894159 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1558,6 +1558,7 @@ class DatacheckEntry: -----------------------+-------------------------------------------- BAD_ANGLE | BUS_WITH_I | + DISCONNECTED_ROUTE | adjacent root's expected connection point DUPLICATE_COORDS | coordinate pair DUPLICATE_LABEL | HIDDEN_JUNCTION | number of incident edges in TM master graph @@ -3545,8 +3546,8 @@ def run(self): lines.pop(0) # ignore header line datacheckfps = [] -datacheck_always_error = [ 'BAD_ANGLE', 'DUPLICATE_LABEL', 'HIDDEN_TERMINUS', - 'INVALID_FINAL_CHAR', 'INVALID_FIRST_CHAR', +datacheck_always_error = [ 'BAD_ANGLE', 'DISCONNECTED_ROUTE', 'DUPLICATE_LABEL', + 'HIDDEN_TERMINUS', 'INVALID_FINAL_CHAR', 'INVALID_FIRST_CHAR', 'LABEL_INVALID_CHAR', 'LABEL_PARENS', 'LABEL_SLASHES', 'LABEL_TOO_LONG', 'LABEL_UNDERSCORES', 'LONG_UNDERSCORE', 'MALFORMED_LAT', 'MALFORMED_LON', 'MALFORMED_URL', From 96465fbbc8868ccac214ec25348a33993cb4aaac Mon Sep 17 00:00:00 2001 From: eric bryant Date: Fri, 12 Jun 2020 13:56:33 -0400 Subject: [PATCH 16/34] BUS_WITH_I datacheck improvements Closes #252 * Require at least 1 numeral * Account for [NEWS] suffixes, as in I-35E & I-35W * Case insensitive Bus * Don't require full match; flag cases such as I-95Bus(42) and I-95Bus_N * Python & C++ flavors now behave identically * Only flag errors in the USA --- .../cplusplus/classes/Waypoint/Waypoint.cpp | 10 +++++-- siteupdate/python-teresco/siteupdate.py | 26 ++++++++----------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp index 812f0df1..0373437c 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp @@ -379,10 +379,16 @@ inline void Waypoint::visible_distance(DatacheckEntryList *datacheckerrors, char inline void Waypoint::bus_with_i(DatacheckEntryList *datacheckerrors) { // look for I-xx with Bus instead of BL or BS - if (label[0] != 'I' || label[1] != '-') return; + if (label[0] != 'I' || label[1] != '-' || route->region->country->first != "USA") return; const char *c = label.data()+2; + if (*c < '0' || *c > '9') return; while (*c >= '0' && *c <= '9') c++; - if (!strncmp(c, "Bus", 3)) datacheckerrors->add(route, label, "", "", "BUS_WITH_I", ""); + if ( *c == 'E' || *c == 'W' || *c == 'N' || *c == 'S' + || *c == 'e' || *c == 'w' || *c == 'n' || *c == 's' ) c++; + if ( (*c == 'B' || *c == 'b') + && (*(c+1) == 'u' || *(c+1) == 'U') + && (*(c+2) == 's' || *(c+2) == 'S') ) + datacheckerrors->add(route, label, "", "", "BUS_WITH_I", ""); } inline void Waypoint::label_looks_hidden(DatacheckEntryList *datacheckerrors) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index c6894159..6c496013 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -913,11 +913,7 @@ def __init__(self,line,system,el): ".csv line [" + line + "], expected " + system.systemname) self.region = fields[1] region_found = False - for r in all_regions: - if r[0] == self.region: - region_found = True - break - if not region_found: + if self.region not in all_regions: el.add_error("Unrecognized region in " + system.systemname + ".csv line: " + line) self.route = fields[2] @@ -2538,7 +2534,7 @@ def __init__(self,filename,descr,vertices,edges,travelers,format,category): " bytes in countries.csv line " + line) countries.append(fields) -all_regions = [] +all_regions = {} try: file = open(args.highwaydatapath+"/regions.csv", "rt",encoding='utf-8') except OSError as e: @@ -2580,7 +2576,7 @@ def __init__(self,filename,descr,vertices,edges,travelers,format,category): if len(fields[4].encode('utf-8')) > DBFieldLength.regiontype: el.add_error("Region type > " + str(DBFieldLength.regiontype) + " bytes in regions.csv line " + line) - all_regions.append(fields) + all_regions[fields[0]] = fields # Create a list of HighwaySystem objects, one per system in systems.csv file highway_systems = [] @@ -3656,7 +3652,7 @@ def run(self): # We will create graph data and a graph file for each region that includes # any active or preview systems - for r in all_regions: + for r in all_regions.values(): region_code = r[0] if region_code not in active_preview_mileage_by_region: continue @@ -3762,9 +3758,9 @@ def run(self): print(fields[1] + ' ', end="", flush=True) region_list = [] selected_regions = fields[2].split(",") - for r in all_regions: - if r[0] in selected_regions and r[0] in active_preview_mileage_by_region: - region_list.append(r[0]) + for r in all_regions.keys(): + if r in selected_regions and r in active_preview_mileage_by_region: + region_list.append(r) graph_data.write_subgraphs_tmg(graph_list, args.graphfilepath + "/", fields[1], fields[0], "multiregion", region_list, None, None, all_waypoints) @@ -3779,7 +3775,7 @@ def run(self): print(et.et() + "Creating country graphs.", flush=True) for c in countries: region_list = [] - for r in all_regions: + for r in all_regions.values(): # does it match this country and have routes? if c[0] == r[2] and r[0] in active_preview_mileage_by_region: region_list.append(r[0]) @@ -3800,7 +3796,7 @@ def run(self): print(et.et() + "Creating continent graphs.", flush=True) for c in continents: region_list = [] - for r in all_regions: + for r in all_regions.values(): # does it match this continent and have routes? if c[0] == r[3] and r[0] in active_preview_mileage_by_region: region_list.append(r[0]) @@ -3953,7 +3949,7 @@ def run(self): datacheckerrors.append(DatacheckEntry(r,[w.label],'NONTERMINAL_UNDERSCORE')) # look for I-xx with Bus instead of BL or BS - if re.fullmatch('I\-[0-9]*Bus', w.label): + if re.fullmatch('I\-[0-9]+[EeWwNnSs]?[Bb][Uu][Ss].*', w.label) and all_regions[w.route.region][2] == "USA": datacheckerrors.append(DatacheckEntry(r,[w.label],'BUS_WITH_I')) # look for labels that look like hidden waypoints but @@ -4117,7 +4113,7 @@ def run(self): '), PRIMARY KEY(code), FOREIGN KEY (country) REFERENCES countries(code), FOREIGN KEY (continent) REFERENCES continents(code));\n') sqlfile.write('INSERT INTO regions VALUES\n') first = True - for r in all_regions: + for r in all_regions.values(): if not first: sqlfile.write(",") first = False From cdcfbb5e27d6131e60e595df76ac336d2e9755ed Mon Sep 17 00:00:00 2001 From: eric bryant Date: Fri, 12 Jun 2020 16:43:09 -0400 Subject: [PATCH 17/34] detect BUS_WITH_I for I-##C (e.g. I-69C) --- siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp | 4 ++-- siteupdate/python-teresco/siteupdate.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp index 0373437c..078d91d1 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp @@ -383,8 +383,8 @@ inline void Waypoint::bus_with_i(DatacheckEntryList *datacheckerrors) const char *c = label.data()+2; if (*c < '0' || *c > '9') return; while (*c >= '0' && *c <= '9') c++; - if ( *c == 'E' || *c == 'W' || *c == 'N' || *c == 'S' - || *c == 'e' || *c == 'w' || *c == 'n' || *c == 's' ) c++; + if ( *c == 'E' || *c == 'W' || *c == 'C' || *c == 'N' || *c == 'S' + || *c == 'e' || *c == 'w' || *c == 'c' || *c == 'n' || *c == 's' ) c++; if ( (*c == 'B' || *c == 'b') && (*(c+1) == 'u' || *(c+1) == 'U') && (*(c+2) == 's' || *(c+2) == 'S') ) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 6c496013..fc55b6f5 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -3949,7 +3949,7 @@ def run(self): datacheckerrors.append(DatacheckEntry(r,[w.label],'NONTERMINAL_UNDERSCORE')) # look for I-xx with Bus instead of BL or BS - if re.fullmatch('I\-[0-9]+[EeWwNnSs]?[Bb][Uu][Ss].*', w.label) and all_regions[w.route.region][2] == "USA": + if re.fullmatch('I\-[0-9]+[EeWwCcNnSs]?[Bb][Uu][Ss].*', w.label) and all_regions[w.route.region][2] == "USA": datacheckerrors.append(DatacheckEntry(r,[w.label],'BUS_WITH_I')) # look for labels that look like hidden waypoints but From 0e34dfa4ec24ad749db5f81d3af6fae148e7ce5d Mon Sep 17 00:00:00 2001 From: eric bryant Date: Fri, 12 Jun 2020 19:32:43 -0400 Subject: [PATCH 18/34] invalid char datacheck improvements * Python: prevent crash when entire label is asterisks * C++: flag error when label == "*" --- siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp | 8 ++++++-- siteupdate/python-teresco/siteupdate.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp index 078d91d1..9ec0d0fc 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp @@ -406,13 +406,17 @@ inline void Waypoint::label_looks_hidden(DatacheckEntryList *datacheckerrors) inline void Waypoint::label_invalid_char(DatacheckEntryList *datacheckerrors) { // look for labels with invalid characters - for (const char *c = label.data(); *c; c++) + if (label == "*") + datacheckerrors->add(route, label, "", "", "LABEL_INVALID_CHAR", ""); + else for (const char *c = label.data(); *c; c++) if ((*c == 42 || *c == 43) && c > label.data() || (*c < 40) || (*c == 44) || (*c > 57 && *c < 65) || (*c == 96) || (*c > 122) || (*c > 90 && *c < 95)) datacheckerrors->add(route, label, "", "", "LABEL_INVALID_CHAR", ""); for (std::string& lbl : alt_labels) - for (const char *c = lbl.data(); *c; c++) + if (lbl == "*") + datacheckerrors->add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); + else for (const char *c = lbl.data(); *c; c++) if (*c == '+' && c > lbl.data() || *c == '*' && (c > lbl.data()+1 || lbl[0] != '+') || (*c < 40) || (*c == 44) || (*c > 57 && *c < 65) || (*c == 96) || (*c > 122) || (*c > 90 && *c < 95)) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index fc55b6f5..55668533 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -3936,7 +3936,7 @@ def run(self): # look for labels with invalid first or last character index = 0 - while w.label[index] == '*': + while index < len(w.label) and w.label[index] == '*': index += 1 if index < len(w.label) and w.label[index] in "_/(": datacheckerrors.append(DatacheckEntry(r,[w.label],'INVALID_FIRST_CHAR', w.label[index])) From ec822f0d3d21fe323ed28d8caf32b140b2152ceb Mon Sep 17 00:00:00 2001 From: eric bryant Date: Sun, 14 Jun 2020 14:48:30 -0400 Subject: [PATCH 19/34] LABEL_INVALID_CHAR for all points ...not just visible ones --- siteupdate/cplusplus/classes/Route/read_wpt.cpp | 2 +- siteupdate/python-teresco/siteupdate.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/siteupdate/cplusplus/classes/Route/read_wpt.cpp b/siteupdate/cplusplus/classes/Route/read_wpt.cpp index 4af30af2..7253ece7 100644 --- a/siteupdate/cplusplus/classes/Route/read_wpt.cpp +++ b/siteupdate/cplusplus/classes/Route/read_wpt.cpp @@ -62,6 +62,7 @@ void Route::read_wpt // single-point Datachecks, and HighwaySegment w->out_of_bounds(datacheckerrors, fstr); w->duplicate_coords(datacheckerrors, coords_used, fstr); + w->label_invalid_char(datacheckerrors); if (point_list.size() > 1) { w->distance_update(datacheckerrors, fstr, vis_dist, point_list[point_list.size()-2]); // add HighwaySegment, if not first point @@ -76,7 +77,6 @@ void Route::read_wpt w->label_slashes(datacheckerrors, slash); w->underscore_datachecks(datacheckerrors, slash); w->label_parens(datacheckerrors); - w->label_invalid_char(datacheckerrors); w->label_invalid_ends(datacheckerrors); w->bus_with_i(datacheckerrors); w->label_looks_hidden(datacheckerrors); diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 55668533..769217bb 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -3853,6 +3853,13 @@ def run(self): else: coords_used.add(latlng) + # look for labels with invalid characters + if not re.fullmatch('\+?\*?[a-zA-Z0-9()/_\-\.]+', w.label): + datacheckerrors.append(DatacheckEntry(r,[w.label],'LABEL_INVALID_CHAR')) + for a in w.alt_labels: + if not re.fullmatch('\+?\*?[a-zA-Z0-9()/_\-\.]+', a): + datacheckerrors.append(DatacheckEntry(r,[a],'LABEL_INVALID_CHAR')) + # visible distance update, and last segment length check if prev_w is not None: last_distance = w.distance_to(prev_w) @@ -3927,13 +3934,6 @@ def run(self): or (left_count == 1 and w.label.index('(') > w.label.index(')')): datacheckerrors.append(DatacheckEntry(r,[w.label],'LABEL_PARENS')) - # look for labels with invalid characters - if not re.fullmatch('\*?[a-zA-Z0-9()/_\-\.]+', w.label): - datacheckerrors.append(DatacheckEntry(r,[w.label],'LABEL_INVALID_CHAR')) - for a in w.alt_labels: - if not re.fullmatch('\+?\*?[a-zA-Z0-9()/_\-\.]+', a): - datacheckerrors.append(DatacheckEntry(r,[a],'LABEL_INVALID_CHAR')) - # look for labels with invalid first or last character index = 0 while index < len(w.label) and w.label[index] == '*': From c039a9709e9a10ca3c8d8c6bf6a674d1c99a42e8 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Mon, 15 Jun 2020 16:18:50 -0400 Subject: [PATCH 20/34] LACKS_GENERIC datacheck --- siteupdate/cplusplus/classes/Route/read_wpt.cpp | 1 + siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp | 10 ++++++++++ siteupdate/cplusplus/classes/Waypoint/Waypoint.h | 1 + siteupdate/python-teresco/siteupdate.py | 4 ++++ 4 files changed, 16 insertions(+) diff --git a/siteupdate/cplusplus/classes/Route/read_wpt.cpp b/siteupdate/cplusplus/classes/Route/read_wpt.cpp index 7253ece7..e2c33867 100644 --- a/siteupdate/cplusplus/classes/Route/read_wpt.cpp +++ b/siteupdate/cplusplus/classes/Route/read_wpt.cpp @@ -80,6 +80,7 @@ void Route::read_wpt w->label_invalid_ends(datacheckerrors); w->bus_with_i(datacheckerrors); w->label_looks_hidden(datacheckerrors); + w->lacks_generic(datacheckerrors); } } delete[] wptdata; diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp index 9ec0d0fc..d1836ad6 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp @@ -346,6 +346,16 @@ inline bool Waypoint::label_too_long(DatacheckEntryList *datacheckerrors) return 0; } +inline void Waypoint::lacks_generic(DatacheckEntryList *datacheckerrors) +{ // label lacks generic highway type + const char* c = label[0] == '*' ? label.data()+1 : label.data(); + if ( (*c == 'O' || *c == 'o') + && (*(c+1) == 'l' || *(c+1) == 'L') + && (*(c+2) == 'd' || *(c+2) == 'D') + && *(c+3) >= '0' && *(c+3) <= '9') + datacheckerrors->add(route, label, "", "", "LACKS_GENERIC", ""); +} + inline void Waypoint::out_of_bounds(DatacheckEntryList *datacheckerrors, char *fstr) { // out-of-bounds coords if (lat > 90 || lat < -90 || lng > 180 || lng < -180) diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.h b/siteupdate/cplusplus/classes/Waypoint/Waypoint.h index 9d7ddc2c..f0862cb8 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.h +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.h @@ -52,5 +52,6 @@ class Waypoint inline void label_slashes(DatacheckEntryList *, const char *); inline void label_selfref(DatacheckEntryList *, const char *); inline bool label_too_long(DatacheckEntryList *); + inline void lacks_generic(DatacheckEntryList *); inline void underscore_datachecks(DatacheckEntryList *, const char *); }; diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 769217bb..55dc8ac8 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -3957,6 +3957,10 @@ def run(self): if re.fullmatch('X[0-9][0-9][0-9][0-9][0-9][0-9]', w.label): datacheckerrors.append(DatacheckEntry(r,[w.label],'LABEL_LOOKS_HIDDEN')) + # lacks generic highway type + if re.fullmatch('^\*?[Oo][lL][dD][0-9].*', w.label): + datacheckerrors.append(DatacheckEntry(r,[w.label],'LACKS_GENERIC')) + # look for USxxxA but not USxxxAlt, B/Bus (others?) ##if re.fullmatch('US[0-9]+A.*', w.label) and not re.fullmatch('US[0-9]+Alt.*', w.label) or \ ## re.fullmatch('US[0-9]+B.*', w.label) and \ From c878b32617bff601333bcbc66bbd8b737389f0d9 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Mon, 15 Jun 2020 16:42:00 -0400 Subject: [PATCH 21/34] DUPLICATE_LABEL userlog warning bugfix --- siteupdate/python-teresco/siteupdate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 55dc8ac8..084f314e 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2922,10 +2922,10 @@ def run(self): # create label->index hashes and check if AltLabels duplicated if w.alt_labels[a] in r.pri_label_hash: datacheckerrors.append(DatacheckEntry(r, [w.alt_labels[a]], "DUPLICATE_LABEL")) - r.duplicate_labels.add(a) + r.duplicate_labels.add(w.alt_labels[a]) elif w.alt_labels[a] in r.alt_label_hash: datacheckerrors.append(DatacheckEntry(r, [w.alt_labels[a]], "DUPLICATE_LABEL")) - r.duplicate_labels.add(a) + r.duplicate_labels.add(w.alt_labels[a]) else: r.alt_label_hash[w.alt_labels[a]] = index index += 1 From 0dc0eb98622220b729a68d25865b0ed6082ed970 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Mon, 15 Jun 2020 23:49:07 -0400 Subject: [PATCH 22/34] BUS_WITH_I: *closed points --- siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp | 5 +++-- siteupdate/python-teresco/siteupdate.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp index d1836ad6..e964b9bc 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp @@ -389,8 +389,9 @@ inline void Waypoint::visible_distance(DatacheckEntryList *datacheckerrors, char inline void Waypoint::bus_with_i(DatacheckEntryList *datacheckerrors) { // look for I-xx with Bus instead of BL or BS - if (label[0] != 'I' || label[1] != '-' || route->region->country->first != "USA") return; - const char *c = label.data()+2; + const char *c = label.data(); + if (*c == '*') c++; + if (*c++ != 'I' || *c++ != '-' || route->region->country->first != "USA") return; if (*c < '0' || *c > '9') return; while (*c >= '0' && *c <= '9') c++; if ( *c == 'E' || *c == 'W' || *c == 'C' || *c == 'N' || *c == 'S' diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 084f314e..1ec7e47c 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -3949,7 +3949,7 @@ def run(self): datacheckerrors.append(DatacheckEntry(r,[w.label],'NONTERMINAL_UNDERSCORE')) # look for I-xx with Bus instead of BL or BS - if re.fullmatch('I\-[0-9]+[EeWwCcNnSs]?[Bb][Uu][Ss].*', w.label) and all_regions[w.route.region][2] == "USA": + if re.fullmatch('\*?I\-[0-9]+[EeWwCcNnSs]?[Bb][Uu][Ss].*', w.label) and all_regions[w.route.region][2] == "USA": datacheckerrors.append(DatacheckEntry(r,[w.label],'BUS_WITH_I')) # look for labels that look like hidden waypoints but From d30d3ba511a9e7d08d61d45a60f7c378ab06b32b Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Wed, 1 Jul 2020 13:00:21 -0400 Subject: [PATCH 23/34] http->https in message --- siteupdate/python-teresco/localupdate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteupdate/python-teresco/localupdate.sh b/siteupdate/python-teresco/localupdate.sh index 8e7af241..5e58c56f 100644 --- a/siteupdate/python-teresco/localupdate.sh +++ b/siteupdate/python-teresco/localupdate.sh @@ -121,7 +121,7 @@ mv TravelMapping-$datestr.sql $tmpdir echo "$0: sending email notification" mailx -s "Travel Mapping Site Update Complete" travelmapping-siteupdates@teresco.org < Date: Fri, 7 Aug 2020 12:34:40 -0400 Subject: [PATCH 24/34] remove extraneous condition We've already checked for this condition in the outer if statement just above. --- siteupdate/cplusplus/siteupdate.cpp | 2 +- siteupdate/python-teresco/siteupdate.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index 91612622..33252cf2 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -412,7 +412,7 @@ int main(int argc, char *argv[]) q->is_reversed = 1; r.is_reversed = 1; } - else if ( !q->con_end()->same_coords(r.con_beg()) ) + else { datacheckerrors->add(&r, r.con_beg()->label, "", "", "DISCONNECTED_ROUTE", q->con_end()->root_at_label()); datacheckerrors->add(q, q->con_end()->label, "", "", diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 1ec7e47c..73f1de2d 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2894,7 +2894,7 @@ def run(self): elif q.con_beg().same_coords(r.con_end()): q.is_reversed = True r.is_reversed = True - elif not q.con_end().same_coords(r.con_beg()): + else: datacheckerrors.append(DatacheckEntry(r, [r.con_beg().label], "DISCONNECTED_ROUTE", q.root + '@' + q.con_end().label)) datacheckerrors.append(DatacheckEntry(q, [q.con_end().label], From e7eb26e766eca88ad1477ae3c23da3009f1b22d0 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Sun, 1 Nov 2020 16:49:02 -0500 Subject: [PATCH 25/34] INTERSTATE_NO_HYPHEN and Waypoint datacheck reorganization per https://github.com/TravelMapping/DataProcessing/issues/349 --- .../cplusplus/classes/DatacheckEntry.cpp | 1 + .../cplusplus/classes/Route/read_wpt.cpp | 15 +- .../cplusplus/classes/Waypoint/Waypoint.cpp | 175 +++++++++--------- .../cplusplus/classes/Waypoint/Waypoint.h | 13 +- siteupdate/cplusplus/siteupdate.cpp | 3 +- siteupdate/python-teresco/siteupdate.py | 18 +- 6 files changed, 123 insertions(+), 102 deletions(-) diff --git a/siteupdate/cplusplus/classes/DatacheckEntry.cpp b/siteupdate/cplusplus/classes/DatacheckEntry.cpp index 99f7cd61..5151cf2d 100644 --- a/siteupdate/cplusplus/classes/DatacheckEntry.cpp +++ b/siteupdate/cplusplus/classes/DatacheckEntry.cpp @@ -17,6 +17,7 @@ class DatacheckEntry DUPLICATE_LABEL | HIDDEN_JUNCTION | number of incident edges in TM master graph HIDDEN_TERMINUS | + INTERSTATE_NO_HYPHEN | INVALID_FINAL_CHAR | final character in label INVALID_FIRST_CHAR | first character in label other than * LABEL_INVALID_CHAR | diff --git a/siteupdate/cplusplus/classes/Route/read_wpt.cpp b/siteupdate/cplusplus/classes/Route/read_wpt.cpp index e2c33867..ba51e96f 100644 --- a/siteupdate/cplusplus/classes/Route/read_wpt.cpp +++ b/siteupdate/cplusplus/classes/Route/read_wpt.cpp @@ -71,16 +71,17 @@ void Route::read_wpt } // checks for visible points if (!w->is_hidden) - { w->visible_distance(datacheckerrors, fstr, vis_dist, last_visible); - const char *slash = strchr(w->label.data(), '/'); - w->label_selfref(datacheckerrors, slash); - w->label_slashes(datacheckerrors, slash); - w->underscore_datachecks(datacheckerrors, slash); - w->label_parens(datacheckerrors); - w->label_invalid_ends(datacheckerrors); + { const char *slash = strchr(w->label.data(), '/'); w->bus_with_i(datacheckerrors); + w->interstate_no_hyphen(datacheckerrors); + w->label_invalid_ends(datacheckerrors); w->label_looks_hidden(datacheckerrors); + w->label_parens(datacheckerrors); + w->label_selfref(datacheckerrors, slash); + w->label_slashes(datacheckerrors, slash); w->lacks_generic(datacheckerrors); + w->underscore_datachecks(datacheckerrors, slash); + w->visible_distance(datacheckerrors, fstr, vis_dist, last_visible); } } delete[] wptdata; diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp index e964b9bc..e8d67fe0 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp @@ -305,6 +305,16 @@ bool Waypoint::label_references_route(Route *r, DatacheckEntryList *datacheckerr /* Datacheck */ +inline void Waypoint::distance_update(DatacheckEntryList *datacheckerrors, char *fstr, double &vis_dist, Waypoint *prev_w) +{ // visible distance update, and last segment length check + double last_distance = distance_to(prev_w); + vis_dist += last_distance; + if (last_distance > 20) + { sprintf(fstr, "%.2f", last_distance); + datacheckerrors->add(route, prev_w->label, label, "", "LONG_SEGMENT", fstr); + } +} + inline void Waypoint::duplicate_coords(DatacheckEntryList *datacheckerrors, std::unordered_set &coords_used, char *fstr) { // duplicate coordinates Waypoint *w; @@ -320,6 +330,25 @@ inline void Waypoint::duplicate_coords(DatacheckEntryList *datacheckerrors, std: } } +inline void Waypoint::label_invalid_char(DatacheckEntryList *datacheckerrors) +{ // look for labels with invalid characters + if (label == "*") + datacheckerrors->add(route, label, "", "", "LABEL_INVALID_CHAR", ""); + else for (const char *c = label.data(); *c; c++) + if ((*c == 42 || *c == 43) && c > label.data() + || (*c < 40) || (*c == 44) || (*c > 57 && *c < 65) + || (*c == 96) || (*c > 122) || (*c > 90 && *c < 95)) + datacheckerrors->add(route, label, "", "", "LABEL_INVALID_CHAR", ""); + for (std::string& lbl : alt_labels) + if (lbl == "*") + datacheckerrors->add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); + else for (const char *c = lbl.data(); *c; c++) + if (*c == '+' && c > lbl.data() || *c == '*' && (c > lbl.data()+1 || lbl[0] != '+') + || (*c < 40) || (*c == 44) || (*c > 57 && *c < 65) + || (*c == 96) || (*c > 122) || (*c > 90 && *c < 95)) + datacheckerrors->add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); +} + inline bool Waypoint::label_too_long(DatacheckEntryList *datacheckerrors) { // label longer than the DB can store if (label.size() > DBFieldLength::label) @@ -346,16 +375,6 @@ inline bool Waypoint::label_too_long(DatacheckEntryList *datacheckerrors) return 0; } -inline void Waypoint::lacks_generic(DatacheckEntryList *datacheckerrors) -{ // label lacks generic highway type - const char* c = label[0] == '*' ? label.data()+1 : label.data(); - if ( (*c == 'O' || *c == 'o') - && (*(c+1) == 'l' || *(c+1) == 'L') - && (*(c+2) == 'd' || *(c+2) == 'D') - && *(c+3) >= '0' && *(c+3) <= '9') - datacheckerrors->add(route, label, "", "", "LACKS_GENERIC", ""); -} - inline void Waypoint::out_of_bounds(DatacheckEntryList *datacheckerrors, char *fstr) { // out-of-bounds coords if (lat > 90 || lat < -90 || lng > 180 || lng < -180) @@ -364,34 +383,13 @@ inline void Waypoint::out_of_bounds(DatacheckEntryList *datacheckerrors, char *f } } -inline void Waypoint::distance_update(DatacheckEntryList *datacheckerrors, char *fstr, double &vis_dist, Waypoint *prev_w) -{ // visible distance update, and last segment length check - double last_distance = distance_to(prev_w); - vis_dist += last_distance; - if (last_distance > 20) - { sprintf(fstr, "%.2f", last_distance); - datacheckerrors->add(route, prev_w->label, label, "", "LONG_SEGMENT", fstr); - } -} - /* checks for visible points */ -inline void Waypoint::visible_distance(DatacheckEntryList *datacheckerrors, char *fstr, double &vis_dist, Waypoint *&last_visible) -{ // complete visible distance check, omit report for active - // systems to reduce clutter - if (vis_dist > 10 && !route->system->active()) - { sprintf(fstr, "%.2f", vis_dist); - datacheckerrors->add(route, last_visible->label, label, "", "VISIBLE_DISTANCE", fstr); - } - last_visible = this; - vis_dist = 0; -} - inline void Waypoint::bus_with_i(DatacheckEntryList *datacheckerrors) { // look for I-xx with Bus instead of BL or BS const char *c = label.data(); if (*c == '*') c++; - if (*c++ != 'I' || *c++ != '-' || route->region->country->first != "USA") return; + if (*c++ != 'I' || *c++ != '-' || route->system->country->first != "USA") return; if (*c < '0' || *c > '9') return; while (*c >= '0' && *c <= '9') c++; if ( *c == 'E' || *c == 'W' || *c == 'C' || *c == 'N' || *c == 'S' @@ -402,6 +400,25 @@ inline void Waypoint::bus_with_i(DatacheckEntryList *datacheckerrors) datacheckerrors->add(route, label, "", "", "BUS_WITH_I", ""); } +inline void Waypoint::interstate_no_hyphen(DatacheckEntryList *datacheckerrors) +{ if (route->system->country->first == "USA" && label.size() >= 2) + { const char *c = label.data(); + if (c[0] == 'T' && c[1] == 'o') c += 2; + if (c[0] == 'I' && isdigit(c[1])) + datacheckerrors->add(route, label, "", "", "INTERSTATE_NO_HYPHEN", ""); + } +} + +inline void Waypoint::label_invalid_ends(DatacheckEntryList *datacheckerrors) +{ // look for labels with invalid first or final characters + const char *c = label.data(); + while (*c == '*') c++; + if (*c == '_' || *c == '/' || *c == '(') + datacheckerrors->add(route, label, "", "", "INVALID_FIRST_CHAR", std::string(1, *c)); + if (label.back() == '_' || label.back() == '/') + datacheckerrors->add(route, label, "", "", "INVALID_FINAL_CHAR", std::string(1, label.back())); +} + inline void Waypoint::label_looks_hidden(DatacheckEntryList *datacheckerrors) { // look for labels that look like hidden waypoints but which aren't hidden if (label.size() != 7) return; @@ -415,35 +432,6 @@ inline void Waypoint::label_looks_hidden(DatacheckEntryList *datacheckerrors) datacheckerrors->add(route, label, "", "", "LABEL_LOOKS_HIDDEN", ""); } -inline void Waypoint::label_invalid_char(DatacheckEntryList *datacheckerrors) -{ // look for labels with invalid characters - if (label == "*") - datacheckerrors->add(route, label, "", "", "LABEL_INVALID_CHAR", ""); - else for (const char *c = label.data(); *c; c++) - if ((*c == 42 || *c == 43) && c > label.data() - || (*c < 40) || (*c == 44) || (*c > 57 && *c < 65) - || (*c == 96) || (*c > 122) || (*c > 90 && *c < 95)) - datacheckerrors->add(route, label, "", "", "LABEL_INVALID_CHAR", ""); - for (std::string& lbl : alt_labels) - if (lbl == "*") - datacheckerrors->add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); - else for (const char *c = lbl.data(); *c; c++) - if (*c == '+' && c > lbl.data() || *c == '*' && (c > lbl.data()+1 || lbl[0] != '+') - || (*c < 40) || (*c == 44) || (*c > 57 && *c < 65) - || (*c == 96) || (*c > 122) || (*c > 90 && *c < 95)) - datacheckerrors->add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); -} - -inline void Waypoint::label_invalid_ends(DatacheckEntryList *datacheckerrors) -{ // look for labels with invalid first or final characters - const char *c = label.data(); - while (*c == '*') c++; - if (*c == '_' || *c == '/' || *c == '(') - datacheckerrors->add(route, label, "", "", "INVALID_FIRST_CHAR", std::string(1, *c)); - if (label.back() == '_' || label.back() == '/') - datacheckerrors->add(route, label, "", "", "INVALID_FINAL_CHAR", std::string(1, label.back())); -} - inline void Waypoint::label_parens(DatacheckEntryList *datacheckerrors) { // look for parenthesis balance in label int parens = 0; @@ -467,28 +455,6 @@ inline void Waypoint::label_parens(DatacheckEntryList *datacheckerrors) datacheckerrors->add(route, label, "", "", "LABEL_PARENS", ""); } -inline void Waypoint::underscore_datachecks(DatacheckEntryList *datacheckerrors, const char *slash) -{ const char *underscore = strchr(label.data(), '_'); - if (underscore) - { // look for too many underscores in label - if (strchr(underscore+1, '_')) - datacheckerrors->add(route, label, "", "", "LABEL_UNDERSCORES", ""); - // look for too many characters after underscore in label - if (label.data()+label.size() > underscore+4) - if (label.back() > 'Z' || label.back() < 'A' || label.data()+label.size() > underscore+5) - datacheckerrors->add(route, label, "", "", "LONG_UNDERSCORE", ""); - // look for labels with a slash after an underscore - if (slash > underscore) - datacheckerrors->add(route, label, "", "", "NONTERMINAL_UNDERSCORE", ""); - } -} - -inline void Waypoint::label_slashes(DatacheckEntryList *datacheckerrors, const char *slash) -{ // look for too many slashes in label - if (slash && strchr(slash+1, '/')) - datacheckerrors->add(route, label, "", "", "LABEL_SLASHES", ""); -} - inline void Waypoint::label_selfref(DatacheckEntryList *datacheckerrors, const char *slash) { // looking for the route within the label //match_start = w.label.find(r.route) @@ -532,8 +498,51 @@ inline void Waypoint::label_selfref(DatacheckEntryList *datacheckerrors, const c datacheckerrors->add(route, label, "", "", "LABEL_SELFREF", ""); } +inline void Waypoint::label_slashes(DatacheckEntryList *datacheckerrors, const char *slash) +{ // look for too many slashes in label + if (slash && strchr(slash+1, '/')) + datacheckerrors->add(route, label, "", "", "LABEL_SLASHES", ""); +} + +inline void Waypoint::lacks_generic(DatacheckEntryList *datacheckerrors) +{ // label lacks generic highway type + const char* c = label[0] == '*' ? label.data()+1 : label.data(); + if ( (*c == 'O' || *c == 'o') + && (*(c+1) == 'l' || *(c+1) == 'L') + && (*(c+2) == 'd' || *(c+2) == 'D') + && *(c+3) >= '0' && *(c+3) <= '9') + datacheckerrors->add(route, label, "", "", "LACKS_GENERIC", ""); +} + +inline void Waypoint::underscore_datachecks(DatacheckEntryList *datacheckerrors, const char *slash) +{ const char *underscore = strchr(label.data(), '_'); + if (underscore) + { // look for too many underscores in label + if (strchr(underscore+1, '_')) + datacheckerrors->add(route, label, "", "", "LABEL_UNDERSCORES", ""); + // look for too many characters after underscore in label + if (label.data()+label.size() > underscore+4) + if (label.back() > 'Z' || label.back() < 'A' || label.data()+label.size() > underscore+5) + datacheckerrors->add(route, label, "", "", "LONG_UNDERSCORE", ""); + // look for labels with a slash after an underscore + if (slash > underscore) + datacheckerrors->add(route, label, "", "", "NONTERMINAL_UNDERSCORE", ""); + } +} + // look for USxxxA but not USxxxAlt, B/Bus (others?) //if re.fullmatch('US[0-9]+A.*', w.label) and not re.fullmatch('US[0-9]+Alt.*', w.label) or \ // re.fullmatch('US[0-9]+B.*', w.label) and \ // not (re.fullmatch('US[0-9]+Bus.*', w.label) or re.fullmatch('US[0-9]+Byp.*', w.label)): // datacheckerrors.append(DatacheckEntry(r,[w.label],'US_BANNER')) + +inline void Waypoint::visible_distance(DatacheckEntryList *datacheckerrors, char *fstr, double &vis_dist, Waypoint *&last_visible) +{ // complete visible distance check, omit report for active + // systems to reduce clutter + if (vis_dist > 10 && !route->system->active()) + { sprintf(fstr, "%.2f", vis_dist); + datacheckerrors->add(route, last_visible->label, label, "", "VISIBLE_DISTANCE", fstr); + } + last_visible = this; + vis_dist = 0; +} diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.h b/siteupdate/cplusplus/classes/Waypoint/Waypoint.h index f0862cb8..23a5d09f 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.h +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.h @@ -39,19 +39,20 @@ class Waypoint bool label_references_route(Route *, DatacheckEntryList *); // Datacheck + inline void distance_update(DatacheckEntryList *, char *, double &, Waypoint *); inline void duplicate_coords(DatacheckEntryList *, std::unordered_set &, char *); + inline void label_invalid_char(DatacheckEntryList *); + inline bool label_too_long(DatacheckEntryList *); inline void out_of_bounds(DatacheckEntryList *, char *); - inline void distance_update(DatacheckEntryList *, char *, double &, Waypoint *); // checks for visible points - inline void visible_distance(DatacheckEntryList *, char *, double &, Waypoint *&); inline void bus_with_i(DatacheckEntryList *); - inline void label_looks_hidden(DatacheckEntryList *); - inline void label_invalid_char(DatacheckEntryList *); + inline void interstate_no_hyphen(DatacheckEntryList *); inline void label_invalid_ends(DatacheckEntryList *); + inline void label_looks_hidden(DatacheckEntryList *); inline void label_parens(DatacheckEntryList *); - inline void label_slashes(DatacheckEntryList *, const char *); inline void label_selfref(DatacheckEntryList *, const char *); - inline bool label_too_long(DatacheckEntryList *); + inline void label_slashes(DatacheckEntryList *, const char *); inline void lacks_generic(DatacheckEntryList *); inline void underscore_datachecks(DatacheckEntryList *, const char *); + inline void visible_distance(DatacheckEntryList *, char *, double &, Waypoint *&); }; diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index 33252cf2..fa63f0fc 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -930,7 +930,8 @@ int main(int argc, char *argv[]) list> datacheckfps; unordered_set datacheck_always_error ({ "BAD_ANGLE", "DISCONNECTED_ROUTE", "DUPLICATE_LABEL", - "HIDDEN_TERMINUS", "INVALID_FINAL_CHAR", "INVALID_FIRST_CHAR", + "HIDDEN_TERMINUS", "INTERSTATE_NO_HYPHEN", + "INVALID_FINAL_CHAR", "INVALID_FIRST_CHAR", "LABEL_INVALID_CHAR", "LABEL_PARENS", "LABEL_SLASHES", "LABEL_TOO_LONG", "LABEL_UNDERSCORES", "LONG_UNDERSCORE", "MALFORMED_LAT", "MALFORMED_LON", "MALFORMED_URL", diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 73f1de2d..a8d99ad9 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1559,6 +1559,7 @@ class DatacheckEntry: DUPLICATE_LABEL | HIDDEN_JUNCTION | number of incident edges in TM master graph HIDDEN_TERMINUS | + INTERSTATE_NO_HYPHEN | INVALID_FINAL_CHAR | final character in label INVALID_FIRST_CHAR | first character in label other than * LABEL_INVALID_CHAR | @@ -3543,7 +3544,8 @@ def run(self): lines.pop(0) # ignore header line datacheckfps = [] datacheck_always_error = [ 'BAD_ANGLE', 'DISCONNECTED_ROUTE', 'DUPLICATE_LABEL', - 'HIDDEN_TERMINUS', 'INVALID_FINAL_CHAR', 'INVALID_FIRST_CHAR', + 'HIDDEN_TERMINUS', 'INTERSTATE_NO_HYPHEN', + 'INVALID_FINAL_CHAR', 'INVALID_FIRST_CHAR', 'LABEL_INVALID_CHAR', 'LABEL_PARENS', 'LABEL_SLASHES', 'LABEL_TOO_LONG', 'LABEL_UNDERSCORES', 'LONG_UNDERSCORE', 'MALFORMED_LAT', 'MALFORMED_LON', 'MALFORMED_URL', @@ -3948,10 +3950,6 @@ def run(self): w.label.index('/') > w.label.index('_'): datacheckerrors.append(DatacheckEntry(r,[w.label],'NONTERMINAL_UNDERSCORE')) - # look for I-xx with Bus instead of BL or BS - if re.fullmatch('\*?I\-[0-9]+[EeWwCcNnSs]?[Bb][Uu][Ss].*', w.label) and all_regions[w.route.region][2] == "USA": - datacheckerrors.append(DatacheckEntry(r,[w.label],'BUS_WITH_I')) - # look for labels that look like hidden waypoints but # which aren't hidden if re.fullmatch('X[0-9][0-9][0-9][0-9][0-9][0-9]', w.label): @@ -3961,6 +3959,16 @@ def run(self): if re.fullmatch('^\*?[Oo][lL][dD][0-9].*', w.label): datacheckerrors.append(DatacheckEntry(r,[w.label],'LACKS_GENERIC')) + # USA-only datachecks + if w.route.system.country == "USA" and len(w.label) >= 2: + # look for I-xx with Bus instead of BL or BS + if re.fullmatch('\*?I\-[0-9]+[EeWwCcNnSs]?[Bb][Uu][Ss].*', w.label): + datacheckerrors.append(DatacheckEntry(r,[w.label],'BUS_WITH_I')) + # look for Ixx without hyphen + c = 2 if (w.label.startswith("To") and len(w.label) > 2) else 0 + if w.label[c] == 'I' and w.label[c+1].isdigit(): + datacheckerrors.append(DatacheckEntry(r,[w.label],'INTERSTATE_NO_HYPHEN')) + # look for USxxxA but not USxxxAlt, B/Bus (others?) ##if re.fullmatch('US[0-9]+A.*', w.label) and not re.fullmatch('US[0-9]+Alt.*', w.label) or \ ## re.fullmatch('US[0-9]+B.*', w.label) and \ From 76f9e359bf2852d8c8bd8c44ba1b731ecb74993a Mon Sep 17 00:00:00 2001 From: eric bryant Date: Thu, 12 Nov 2020 23:53:29 -0500 Subject: [PATCH 26/34] prevent empty routes from crashing DISCONNECTED_ROUTE datacheck --- siteupdate/cplusplus/siteupdate.cpp | 2 +- siteupdate/python-teresco/siteupdate.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index fa63f0fc..21b544d5 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -394,7 +394,7 @@ int main(int argc, char *argv[]) // check for mismatched route endpoints within connected routes #define q r.con_route->roots[r.rootOrder-1] - if ( r.rootOrder > 0 && q->point_list.size() > 1 && !r.con_beg()->same_coords(q->con_end()) ) + if ( r.rootOrder > 0 && q->point_list.size() > 1 && r.point_list.size() > 1 && !r.con_beg()->same_coords(q->con_end()) ) { if ( q->con_beg()->same_coords(r.con_beg()) ) { //std::cout << "DEBUG: marking only " << q->str() << " reversed" << std::endl; //if (q->is_reversed) std::cout << "DEBUG: " << q->str() << " already reversed!" << std::endl; diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index a8d99ad9..52cd5bb3 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2887,7 +2887,7 @@ def run(self): # check for mismatched route endpoints within connected routes q = r.con_route.roots[r.rootOrder-1] - if r.rootOrder > 0 and len(q.point_list) > 1 and not r.con_beg().same_coords(q.con_end()): + if r.rootOrder > 0 and len(q.point_list) > 1 and len(r.point_list) > 1 and not r.con_beg().same_coords(q.con_end()): if q.con_beg().same_coords(r.con_beg()): q.is_reversed = True elif q.con_end().same_coords(r.con_end()): From 3d56bd9b39ac06ba810e0bda669df6eb117f63f4 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Fri, 13 Nov 2020 01:54:43 -0500 Subject: [PATCH 27/34] reduce Python<->C++ siteupdate.log diffs --- siteupdate/cplusplus/classes/HighwaySystem.cpp | 2 ++ siteupdate/cplusplus/classes/Route/read_wpt.cpp | 2 +- siteupdate/cplusplus/siteupdate.cpp | 4 +--- siteupdate/python-teresco/siteupdate.py | 12 +++++------- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/siteupdate/cplusplus/classes/HighwaySystem.cpp b/siteupdate/cplusplus/classes/HighwaySystem.cpp index e63f63c1..95dca4ca 100644 --- a/siteupdate/cplusplus/classes/HighwaySystem.cpp +++ b/siteupdate/cplusplus/classes/HighwaySystem.cpp @@ -75,6 +75,8 @@ class HighwaySystem if (level_str != "active" && level_str != "preview" && level_str != "devel") el.add_error("Unrecognized level in " + systemsfile + " line: " + line); + std::cout << systemname << '.' << std::flush; + // read chopped routes CSV file.open(path+"/"+systemname+".csv"); if (!file) el.add_error("Could not open "+path+"/"+systemname+".csv"); diff --git a/siteupdate/cplusplus/classes/Route/read_wpt.cpp b/siteupdate/cplusplus/classes/Route/read_wpt.cpp index ba51e96f..0fe0e1cb 100644 --- a/siteupdate/cplusplus/classes/Route/read_wpt.cpp +++ b/siteupdate/cplusplus/classes/Route/read_wpt.cpp @@ -12,7 +12,7 @@ void Route::read_wpt std::vector lines; std::ifstream file(filename); if (!file) - { el->add_error("Could not open " + filename); + { el->add_error("[Errno 2] No such file or directory: '" + filename + '\''); file.close(); return; } diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index 21b544d5..87d81443 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -244,7 +244,6 @@ int main(int argc, char *argv[]) // deleted on termination of program if (!hs->is_valid) delete hs; else { highway_systems.push_back(hs); - cout << hs->systemname << '.' << std::flush; } } cout << endl; @@ -999,8 +998,7 @@ int main(int argc, char *argv[]) } fpfile.close(); cout << '!' << endl; - cout << et.et() << "Found " << datacheckerrors->entries.size() << " datacheck errors." << endl; - cout << et.et() << "Matched " << fpcount << " FP entries." << endl; + cout << et.et() << "Found " << datacheckerrors->entries.size() << " datacheck errors and matched " << fpcount << " FP entries." << endl; // write log of unmatched false positives from the datacheckfps.csv cout << et.et() << "Writing log of unmatched datacheck FP entries." << endl; diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 52cd5bb3..45f914e0 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1045,6 +1045,7 @@ def read_wpt(self,all_waypoints,all_waypoints_lock,datacheckerrors,el,path="../. self.segment_list.append(HighwaySegment(previous_point, w, self)) if len(self.point_list) < 2: el.add_error("Route contains fewer than 2 points: " + str(self)) + print(".", end="",flush=True) def print_route(self): for point in self.point_list: @@ -2581,7 +2582,7 @@ def __init__(self,filename,descr,vertices,edges,travelers,format,category): # Create a list of HighwaySystem objects, one per system in systems.csv file highway_systems = [] -print(et.et() + "Reading systems list in " + args.highwaydatapath+"/"+args.systemsfile + ". ",flush=True) +print(et.et() + "Reading systems list in " + args.highwaydatapath+"/"+args.systemsfile + ".",flush=True) try: file = open(args.highwaydatapath+"/"+args.systemsfile, "rt",encoding='utf-8') except OSError as e: @@ -2676,7 +2677,6 @@ def read_wpts_for_highway_system(h): all_wpt_files.remove(wpt_path) r.read_wpt(all_waypoints,all_waypoints_lock,datacheckerrors, el,args.highwaydatapath+"/hwy_data") - print(".", end="",flush=True) #print(str(r)) #r.print_route() print("!", flush=True) @@ -2960,7 +2960,7 @@ def run(self): # Read updates.csv file, just keep in the fields array for now since we're # just going to drop this into the DB later anyway updates = [] -print(et.et() + "Reading updates file. ",end="",flush=True) +print(et.et() + "Reading updates file.",end="",flush=True) with open(args.highwaydatapath+"/updates.csv", "rt", encoding='UTF-8') as file: lines = file.readlines() file.close() @@ -2998,7 +2998,7 @@ def run(self): # array for now since we're just going to drop this into the DB later # anyway systemupdates = [] -print(et.et() + "Reading systemupdates file. ",end="",flush=True) +print(et.et() + "Reading systemupdates file.",end="",flush=True) with open(args.highwaydatapath+"/systemupdates.csv", "rt", encoding='UTF-8') as file: lines = file.readlines() file.close() @@ -3997,7 +3997,6 @@ def run(self): datacheckerrors.append(DatacheckEntry(r,labels,'SHARP_ANGLE', "{0:.2f}".format(angle))) print("!", flush=True) -print(et.et() + "Found " + str(len(datacheckerrors)) + " datacheck errors.") datacheckerrors.sort(key=lambda DatacheckEntry: str(DatacheckEntry)) @@ -4005,7 +4004,6 @@ def run(self): print(et.et() + "Marking datacheck false positives.",end="",flush=True) fpfile = open(args.logfilepath+'/nearmatchfps.log','w',encoding='utf-8') fpfile.write("Log file created at: " + str(datetime.datetime.now()) + "\n") -toremove = [] counter = 0 fpcount = 0 for d in datacheckerrors: @@ -4026,7 +4024,7 @@ def run(self): fpfile.write("CHANGETO: " + fp[0] + ';' + fp[1] + ';' + fp[2] + ';' + fp[3] + ';' + fp[4] + ';' + d.info + '\n') fpfile.close() print("!", flush=True) -print(et.et() + "Matched " + str(fpcount) + " FP entries.", flush=True) +print(et.et() + "Found " + str(len(datacheckerrors)) + " datacheck errors and matched " + str(fpcount) + " FP entries.", flush=True) # write log of unmatched false positives from the datacheckfps.csv print(et.et() + "Writing log of unmatched datacheck FP entries.", flush=True) From 53ca19de5505531fb0f32aec78983cbe7b4d74e9 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Sun, 15 Nov 2020 01:16:21 -0500 Subject: [PATCH 28/34] con_total_miles cleanup --- .../cplusplus/classes/TravelerList/userlog.cpp | 13 +++---------- siteupdate/cplusplus/siteupdate.cpp | 8 +++----- siteupdate/python-teresco/siteupdate.py | 12 ++++-------- 3 files changed, 10 insertions(+), 23 deletions(-) diff --git a/siteupdate/cplusplus/classes/TravelerList/userlog.cpp b/siteupdate/cplusplus/classes/TravelerList/userlog.cpp index 5e35dfa3..aa42d40e 100644 --- a/siteupdate/cplusplus/classes/TravelerList/userlog.cpp +++ b/siteupdate/cplusplus/classes/TravelerList/userlog.cpp @@ -83,8 +83,7 @@ void TravelerList::userlog unsigned int num_con_rtes_clinched = 0; log << "System " << h->systemname << " by route (traveled routes only):\n"; for (ConnectedRoute &cr : h->con_route_list) - { double con_total_miles = 0; - double con_clinched_miles = 0; + { double con_clinched_miles = 0; std::string to_write = ""; for (Route *r : cr.roots) { // find traveled mileage on this by this user @@ -99,16 +98,11 @@ void TravelerList::userlog con_clinched_miles += miles; to_write += " " + r->readable_name() + ": " + format_clinched_mi(miles,r->mileage) + "\n"; } - con_total_miles += r->mileage; } if (con_clinched_miles) { system_con_umap[&cr] = con_clinched_miles; char clinched = '0'; - /*yDEBUG - if (traveler_name == "oscar" && cr.system->systemname == "usaus" && cr.route == "US85") - printf("\nOscar on US85:\ncon_clinched_miles = %.17f\n con_total_miles = %.17f\n", - con_clinched_miles, con_total_miles);//*/ - if (con_clinched_miles == con_total_miles) + if (con_clinched_miles == cr.mileage) { num_con_rtes_clinched++; clinched = '1'; } @@ -116,7 +110,7 @@ void TravelerList::userlog if (!strchr(fstr, '.')) strcat(fstr, ".0"); clin_db_val->add_ccr("('" + cr.roots[0]->root + "','" + traveler_name + "','" + std::string(fstr) + "','" + clinched + "')"); - log << cr.readable_name() << ": " << format_clinched_mi(con_clinched_miles, con_total_miles) << '\n'; + log << cr.readable_name() << ": " << format_clinched_mi(con_clinched_miles, cr.mileage) << '\n'; if (cr.roots.size() == 1) log << " (" << cr.roots[0]->readable_name() << " only)\n"; else log << to_write << '\n'; @@ -128,7 +122,6 @@ void TravelerList::userlog num_con_rtes_clinched, (int)h->con_route_list.size(), 100*(double)num_con_rtes_clinched/h->con_route_list.size()); log << "System " << h->systemname << fstr << '\n'; con_routes_traveled[h] = system_con_umap; - //#include "debug/oscars_usaus_ConRtes.cpp" } } diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index 87d81443..76758643 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -765,15 +765,13 @@ int main(int argc, char *argv[]) } hdstatsfile << "System " << h->systemname << " by route:\n"; for (ConnectedRoute& cr : h->con_route_list) - { double con_total_miles = 0; - string to_write = ""; + { string to_write = ""; for (Route *r : cr.roots) { sprintf(fstr, ": %.2f mi\n", r->mileage); to_write += " " + r->readable_name() + fstr; - con_total_miles += r->mileage; + cr.mileage += r->mileage; } - cr.mileage = con_total_miles; //FIXME? - sprintf(fstr, ": %.2f mi", con_total_miles); + sprintf(fstr, ": %.2f mi", cr.mileage); hdstatsfile << cr.readable_name() << fstr; if (cr.roots.size() == 1) hdstatsfile << " (" << cr.roots[0]->readable_name() << " only)\n"; diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 45f914e0..019fe52d 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -3294,13 +3294,11 @@ def run(self): hdstatsfile.write(region + ": " + "{0:.2f}".format(h.mileage_by_region[region]) + " mi\n") hdstatsfile.write("System " + h.systemname + " by route:\n") for cr in h.con_route_list: - con_total_miles = 0.0 to_write = "" for r in cr.roots: to_write += " " + r.readable_name() + ": " + "{0:.2f}".format(r.mileage) + " mi\n" - con_total_miles += r.mileage - cr.mileage = con_total_miles - hdstatsfile.write(cr.readable_name() + ": " + "{0:.2f}".format(con_total_miles) + " mi") + cr.mileage += r.mileage + hdstatsfile.write(cr.readable_name() + ": " + "{0:.2f}".format(cr.mileage) + " mi") if len(cr.roots) == 1: hdstatsfile.write(" (" + cr.roots[0].readable_name() + " only)\n") else: @@ -3403,7 +3401,6 @@ def run(self): con_routes_clinched = 0 t.log_entries.append("System " + h.systemname + " by route (traveled routes only):") for cr in h.con_route_list: - con_total_miles = 0.0 con_clinched_miles = 0.0 to_write = "" for r in cr.roots: @@ -3420,18 +3417,17 @@ def run(self): con_clinched_miles += miles to_write += " " + r.readable_name() + ": " + \ format_clinched_mi(miles,r.mileage) + "\n" - con_total_miles += r.mileage if con_clinched_miles > 0: system_con_dict[cr] = con_clinched_miles clinched = '0' - if con_clinched_miles == con_total_miles: + if con_clinched_miles == cr.mileage: con_routes_clinched += 1 clinched = '1' ccr_values.append("('" + cr.roots[0].root + "','" + t.traveler_name + "','" + str(con_clinched_miles) + "','" + clinched + "')") t.log_entries.append(cr.readable_name() + ": " + \ - format_clinched_mi(con_clinched_miles,con_total_miles)) + format_clinched_mi(con_clinched_miles,cr.mileage)) if len(cr.roots) == 1: t.log_entries.append(" (" + cr.roots[0].readable_name() + " only)") else: From bb1a391a969ce9d2aaa123e26a8dbfd106742125 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Sun, 15 Nov 2020 01:54:53 -0500 Subject: [PATCH 29/34] C++ nmp_merged progress indication --- siteupdate/cplusplus/siteupdate.cpp | 10 +++++++--- siteupdate/cplusplus/threads/NmpMergedThread.cpp | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index 76758643..5e0aebe7 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -365,7 +365,7 @@ int main(int argc, char *argv[]) // if requested, rewrite data with near-miss points merged in if (args.nmpmergepath != "" && !args.errorcheck) - { cout << et.et() << "Writing near-miss point merged wpt files." << endl; //FIXME output dots to indicate progress + { cout << et.et() << "Writing near-miss point merged wpt files." << endl; #ifdef threading_enabled // set up for threaded nmp_merged file writes hs_it = highway_systems.begin(); @@ -377,9 +377,13 @@ int main(int argc, char *argv[]) delete thr[t];//*/ #else for (HighwaySystem *h : highway_systems) - for (Route &r : h->route_list) - r.write_nmp_merged(args.nmpmergepath + "/" + r.rg_str); + { std::cout << h->systemname << std::flush; + for (Route &r : h->route_list) + r.write_nmp_merged(args.nmpmergepath + "/" + r.rg_str); + std::cout << '.' << std::flush; + } #endif + cout << endl; } #include "functions/concurrency_detection.cpp" diff --git a/siteupdate/cplusplus/threads/NmpMergedThread.cpp b/siteupdate/cplusplus/threads/NmpMergedThread.cpp index 2599e7a6..1ddeaf67 100644 --- a/siteupdate/cplusplus/threads/NmpMergedThread.cpp +++ b/siteupdate/cplusplus/threads/NmpMergedThread.cpp @@ -11,6 +11,7 @@ void NmpMergedThread(unsigned int id, std::list *hs_list, std::l (*it)++; //printf("NmpMergedThread %02i (*it)++\n", id); fflush(stdout); mtx->unlock(); + std::cout << h->systemname << '.' << std::flush; for (Route &r : h->route_list) r.write_nmp_merged(*nmpmergepath + "/" + r.rg_str); } From d27abbfaea6a12e95f6deac021e53012d8a1e5df Mon Sep 17 00:00:00 2001 From: eric bryant Date: Sun, 15 Nov 2020 14:00:37 -0500 Subject: [PATCH 30/34] consolidate USA-only datachecks to eliminate redundant string comparisons --- siteupdate/cplusplus/classes/Route/Route.h | 2 +- siteupdate/cplusplus/classes/Route/read_wpt.cpp | 8 +++++--- siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp | 12 +++++------- siteupdate/cplusplus/siteupdate.cpp | 3 ++- siteupdate/cplusplus/threads/ReadWptThread.cpp | 3 ++- siteupdate/python-teresco/siteupdate.py | 3 ++- 6 files changed, 17 insertions(+), 14 deletions(-) diff --git a/siteupdate/cplusplus/classes/Route/Route.h b/siteupdate/cplusplus/classes/Route/Route.h index e099d6db..3f6a4a8d 100644 --- a/siteupdate/cplusplus/classes/Route/Route.h +++ b/siteupdate/cplusplus/classes/Route/Route.h @@ -65,7 +65,7 @@ class Route Route(std::string &, HighwaySystem *, ErrorList &, std::unordered_map &); std::string str(); - void read_wpt(WaypointQuadtree *, ErrorList *, std::string, DatacheckEntryList *, std::unordered_set *); + void read_wpt(WaypointQuadtree *, ErrorList *, std::string, bool, DatacheckEntryList *, std::unordered_set *); void print_route(); HighwaySegment* find_segment_by_waypoints(Waypoint*, Waypoint*); std::string chopped_rtes_line(); diff --git a/siteupdate/cplusplus/classes/Route/read_wpt.cpp b/siteupdate/cplusplus/classes/Route/read_wpt.cpp index 0fe0e1cb..37865962 100644 --- a/siteupdate/cplusplus/classes/Route/read_wpt.cpp +++ b/siteupdate/cplusplus/classes/Route/read_wpt.cpp @@ -1,5 +1,5 @@ void Route::read_wpt -( WaypointQuadtree *all_waypoints, ErrorList *el, std::string path, +( WaypointQuadtree *all_waypoints, ErrorList *el, std::string path, bool usa_flag, DatacheckEntryList *datacheckerrors, std::unordered_set *all_wpt_files ) { /* read data into the Route's waypoint list from a .wpt file */ @@ -72,8 +72,10 @@ void Route::read_wpt // checks for visible points if (!w->is_hidden) { const char *slash = strchr(w->label.data(), '/'); - w->bus_with_i(datacheckerrors); - w->interstate_no_hyphen(datacheckerrors); + if (usa_flag && w->label.size() >= 2) + { w->bus_with_i(datacheckerrors); + w->interstate_no_hyphen(datacheckerrors); + } w->label_invalid_ends(datacheckerrors); w->label_looks_hidden(datacheckerrors); w->label_parens(datacheckerrors); diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp index e8d67fe0..14af1284 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp @@ -389,7 +389,7 @@ inline void Waypoint::bus_with_i(DatacheckEntryList *datacheckerrors) { // look for I-xx with Bus instead of BL or BS const char *c = label.data(); if (*c == '*') c++; - if (*c++ != 'I' || *c++ != '-' || route->system->country->first != "USA") return; + if (*c++ != 'I' || *c++ != '-') return; if (*c < '0' || *c > '9') return; while (*c >= '0' && *c <= '9') c++; if ( *c == 'E' || *c == 'W' || *c == 'C' || *c == 'N' || *c == 'S' @@ -401,12 +401,10 @@ inline void Waypoint::bus_with_i(DatacheckEntryList *datacheckerrors) } inline void Waypoint::interstate_no_hyphen(DatacheckEntryList *datacheckerrors) -{ if (route->system->country->first == "USA" && label.size() >= 2) - { const char *c = label.data(); - if (c[0] == 'T' && c[1] == 'o') c += 2; - if (c[0] == 'I' && isdigit(c[1])) - datacheckerrors->add(route, label, "", "", "INTERSTATE_NO_HYPHEN", ""); - } +{ const char *c = label.data(); + if (c[0] == 'T' && c[1] == 'o') c += 2; + if (c[0] == 'I' && isdigit(c[1])) + datacheckerrors->add(route, label, "", "", "INTERSTATE_NO_HYPHEN", ""); } inline void Waypoint::label_invalid_ends(DatacheckEntryList *datacheckerrors) diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index 87d81443..e8f3c7f3 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -285,8 +285,9 @@ int main(int argc, char *argv[]) #else for (HighwaySystem* h : highway_systems) { std::cout << h->systemname << std::flush; + bool usa_flag = h->country->first == "USA"; for (Route& r : h->route_list) - r.read_wpt(&all_waypoints, &el, args.highwaydatapath+"/hwy_data", datacheckerrors, &all_wpt_files); + r.read_wpt(&all_waypoints, &el, args.highwaydatapath+"/hwy_data", usa_flag, datacheckerrors, &all_wpt_files); std::cout << "!" << std::endl; } #endif diff --git a/siteupdate/cplusplus/threads/ReadWptThread.cpp b/siteupdate/cplusplus/threads/ReadWptThread.cpp index 712111f7..f8a20099 100644 --- a/siteupdate/cplusplus/threads/ReadWptThread.cpp +++ b/siteupdate/cplusplus/threads/ReadWptThread.cpp @@ -16,8 +16,9 @@ void ReadWptThread //printf("ReadWptThread %02i (*it)++\n", id); fflush(stdout); hs_mtx->unlock(); std::cout << h->systemname << std::flush; + bool usa_flag = h->country->first == "USA"; for (Route &r : h->route_list) - r.read_wpt(all_waypoints, el, path, datacheckerrors, all_wpt_files); + r.read_wpt(all_waypoints, el, path, usa_flag, datacheckerrors, all_wpt_files); std::cout << "!" << std::endl; } } diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 45f914e0..024ddc8d 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -3817,6 +3817,7 @@ def run(self): # perform most datachecks here (list initialized above) for h in highway_systems: print(".",end="",flush=True) + usa_flag = h.country == "USA" for r in h.route_list: # set of tuples to be used for finding duplicate coordinates coords_used = set() @@ -3960,7 +3961,7 @@ def run(self): datacheckerrors.append(DatacheckEntry(r,[w.label],'LACKS_GENERIC')) # USA-only datachecks - if w.route.system.country == "USA" and len(w.label) >= 2: + if usa_flag and len(w.label) >= 2: # look for I-xx with Bus instead of BL or BS if re.fullmatch('\*?I\-[0-9]+[EeWwCcNnSs]?[Bb][Uu][Ss].*', w.label): datacheckerrors.append(DatacheckEntry(r,[w.label],'BUS_WITH_I')) From a7a9efd918803479397fec91f000b053c1660864 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Sun, 15 Nov 2020 14:17:44 -0500 Subject: [PATCH 31/34] US_LETTER datacheck --- siteupdate/cplusplus/classes/DatacheckEntry.cpp | 2 +- siteupdate/cplusplus/classes/Route/read_wpt.cpp | 1 + .../cplusplus/classes/Waypoint/Waypoint.cpp | 15 ++++++++++----- siteupdate/cplusplus/classes/Waypoint/Waypoint.h | 1 + siteupdate/cplusplus/siteupdate.cpp | 2 +- siteupdate/python-teresco/siteupdate.py | 16 ++++++++-------- 6 files changed, 22 insertions(+), 15 deletions(-) diff --git a/siteupdate/cplusplus/classes/DatacheckEntry.cpp b/siteupdate/cplusplus/classes/DatacheckEntry.cpp index 5151cf2d..3d781d27 100644 --- a/siteupdate/cplusplus/classes/DatacheckEntry.cpp +++ b/siteupdate/cplusplus/classes/DatacheckEntry.cpp @@ -36,7 +36,7 @@ class DatacheckEntry NONTERMINAL_UNDERSCORE | OUT_OF_BOUNDS | coordinate pair SHARP_ANGLE | angle in degrees - US_BANNER | + US_LETTER | VISIBLE_DISTANCE | distance in miles VISIBLE_HIDDEN_COLOC | hidden point at same coordinates diff --git a/siteupdate/cplusplus/classes/Route/read_wpt.cpp b/siteupdate/cplusplus/classes/Route/read_wpt.cpp index 37865962..4377c79b 100644 --- a/siteupdate/cplusplus/classes/Route/read_wpt.cpp +++ b/siteupdate/cplusplus/classes/Route/read_wpt.cpp @@ -75,6 +75,7 @@ void Route::read_wpt if (usa_flag && w->label.size() >= 2) { w->bus_with_i(datacheckerrors); w->interstate_no_hyphen(datacheckerrors); + w->us_letter(datacheckerrors); } w->label_invalid_ends(datacheckerrors); w->label_looks_hidden(datacheckerrors); diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp index 14af1284..7bd84bdc 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp @@ -528,11 +528,16 @@ inline void Waypoint::underscore_datachecks(DatacheckEntryList *datacheckerrors, } } -// look for USxxxA but not USxxxAlt, B/Bus (others?) -//if re.fullmatch('US[0-9]+A.*', w.label) and not re.fullmatch('US[0-9]+Alt.*', w.label) or \ -// re.fullmatch('US[0-9]+B.*', w.label) and \ -// not (re.fullmatch('US[0-9]+Bus.*', w.label) or re.fullmatch('US[0-9]+Byp.*', w.label)): -// datacheckerrors.append(DatacheckEntry(r,[w.label],'US_BANNER')) +inline void Waypoint::us_letter(DatacheckEntryList *datacheckerrors) +{ // look for USxxxA but not USxxxAlt, B/Bus/Byp + const char* c = label[0] == '*' ? label.data()+1 : label.data(); + if (*c++ != 'U' || *c++ != 'S') return; + if (*c < '0' || *c++ > '9') return; + while (*c >= '0' && *c <= '9') c++; + if (*c < 'A' || *c++ > 'B') return; + if (*c == 0 || *c == '/' || *c == '_' || *c == '(') + datacheckerrors->add(route, label, "", "", "US_LETTER", ""); +} inline void Waypoint::visible_distance(DatacheckEntryList *datacheckerrors, char *fstr, double &vis_dist, Waypoint *&last_visible) { // complete visible distance check, omit report for active diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.h b/siteupdate/cplusplus/classes/Waypoint/Waypoint.h index 23a5d09f..c1ebf0ad 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.h +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.h @@ -54,5 +54,6 @@ class Waypoint inline void label_slashes(DatacheckEntryList *, const char *); inline void lacks_generic(DatacheckEntryList *); inline void underscore_datachecks(DatacheckEntryList *, const char *); + inline void us_letter(DatacheckEntryList *); inline void visible_distance(DatacheckEntryList *, char *, double &, Waypoint *&); }; diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index e8f3c7f3..5490812e 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -935,7 +935,7 @@ int main(int argc, char *argv[]) "LABEL_INVALID_CHAR", "LABEL_PARENS", "LABEL_SLASHES", "LABEL_TOO_LONG", "LABEL_UNDERSCORES", "LONG_UNDERSCORE", "MALFORMED_LAT", "MALFORMED_LON", "MALFORMED_URL", - "NONTERMINAL_UNDERSCORE" + "NONTERMINAL_UNDERSCORE", "US_LETTER" }); while (getline(file, line)) { // trim DOS newlines & trailing whitespace diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 024ddc8d..0aab53f6 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1579,7 +1579,7 @@ class DatacheckEntry: NONTERMINAL_UNDERSCORE | OUT_OF_BOUNDS | coordinate pair SHARP_ANGLE | angle in degrees - US_BANNER | + US_LETTER | VISIBLE_DISTANCE | distance in miles VISIBLE_HIDDEN_COLOC | hidden point at same coordinates @@ -3549,7 +3549,7 @@ def run(self): 'LABEL_INVALID_CHAR', 'LABEL_PARENS', 'LABEL_SLASHES', 'LABEL_TOO_LONG', 'LABEL_UNDERSCORES', 'LONG_UNDERSCORE', 'MALFORMED_LAT', 'MALFORMED_LON', 'MALFORMED_URL', - 'NONTERMINAL_UNDERSCORE' ] + 'NONTERMINAL_UNDERSCORE', 'US_LETTER' ] for line in lines: line=line.strip() if len(line) == 0: @@ -3969,12 +3969,12 @@ def run(self): c = 2 if (w.label.startswith("To") and len(w.label) > 2) else 0 if w.label[c] == 'I' and w.label[c+1].isdigit(): datacheckerrors.append(DatacheckEntry(r,[w.label],'INTERSTATE_NO_HYPHEN')) - - # look for USxxxA but not USxxxAlt, B/Bus (others?) - ##if re.fullmatch('US[0-9]+A.*', w.label) and not re.fullmatch('US[0-9]+Alt.*', w.label) or \ - ## re.fullmatch('US[0-9]+B.*', w.label) and \ - ## not (re.fullmatch('US[0-9]+Bus.*', w.label) or re.fullmatch('US[0-9]+Byp.*', w.label)): - ## datacheckerrors.append(DatacheckEntry(r,[w.label],'US_BANNER')) + # look for USxxxA but not USxxxAlt, B/Bus/Byp + # Eric's paraphrase of Jim's original criteria + # if re.fullmatch('\*?US[0-9]+[AB].*', w.label) and not re.fullmatch('\*?US[0-9]+Alt.*|\*?US[0-9]+Bus.*|\*?US[0-9]+Byp.*', w.label): + # Instead, let's cast a narrower net + if re.fullmatch('\*?US[0-9]+[AB]|\*?US[0-9]+[AB][/_(].*', w.label): + datacheckerrors.append(DatacheckEntry(r,[w.label],'US_LETTER')) prev_w = w From 05faea9c6340f0851be26c05e50f53145842d7ca Mon Sep 17 00:00:00 2001 From: eric bryant Date: Sun, 15 Nov 2020 14:32:58 -0500 Subject: [PATCH 32/34] detect INTERSTATE_NO_HYPHEN in closed waypoints --- siteupdate/cplusplus/classes/DatacheckEntry.cpp | 6 +++--- siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp | 2 +- siteupdate/python-teresco/siteupdate.py | 6 ++++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/siteupdate/cplusplus/classes/DatacheckEntry.cpp b/siteupdate/cplusplus/classes/DatacheckEntry.cpp index 3d781d27..0530a3f3 100644 --- a/siteupdate/cplusplus/classes/DatacheckEntry.cpp +++ b/siteupdate/cplusplus/classes/DatacheckEntry.cpp @@ -3,9 +3,9 @@ class DatacheckEntry route is a pointer to the route with a datacheck error - labels is a list of labels that are related to the error (such - as the endpoints of a too-long segment or the three points that - form a sharp angle) + label1, label2 & label3 are labels that are related to the error + (such as the endpoints of a too-long segment or the three points + that form a sharp angle) code is the error code | info is additional string, one of: | information, if used: diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp index 7bd84bdc..c7efa846 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp @@ -401,7 +401,7 @@ inline void Waypoint::bus_with_i(DatacheckEntryList *datacheckerrors) } inline void Waypoint::interstate_no_hyphen(DatacheckEntryList *datacheckerrors) -{ const char *c = label.data(); +{ const char *c = label[0] == '*' ? label.data()+1 : label.data(); if (c[0] == 'T' && c[1] == 'o') c += 2; if (c[0] == 'I' && isdigit(c[1])) datacheckerrors->add(route, label, "", "", "INTERSTATE_NO_HYPHEN", ""); diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 0aab53f6..820e4817 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -3966,8 +3966,10 @@ def run(self): if re.fullmatch('\*?I\-[0-9]+[EeWwCcNnSs]?[Bb][Uu][Ss].*', w.label): datacheckerrors.append(DatacheckEntry(r,[w.label],'BUS_WITH_I')) # look for Ixx without hyphen - c = 2 if (w.label.startswith("To") and len(w.label) > 2) else 0 - if w.label[c] == 'I' and w.label[c+1].isdigit(): + c = 1 if w.label[0] == '*' else 0 + if w.label[c:c+2] == "To": + c += 2; + if len(w.label) >= c+2 and w.label[c] == 'I' and w.label[c+1].isdigit(): datacheckerrors.append(DatacheckEntry(r,[w.label],'INTERSTATE_NO_HYPHEN')) # look for USxxxA but not USxxxAlt, B/Bus/Byp # Eric's paraphrase of Jim's original criteria From 74b7aea05635eda50065a914ca91b9ff5fb7824e Mon Sep 17 00:00:00 2001 From: eric bryant Date: Sun, 15 Nov 2020 19:07:55 -0500 Subject: [PATCH 33/34] delete vestigial LABEL_SELFREF comments --- siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp | 8 -------- siteupdate/python-teresco/siteupdate.py | 9 --------- 2 files changed, 17 deletions(-) diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp index c7efa846..d1c4ca9e 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp @@ -455,15 +455,7 @@ inline void Waypoint::label_parens(DatacheckEntryList *datacheckerrors) inline void Waypoint::label_selfref(DatacheckEntryList *datacheckerrors, const char *slash) { // looking for the route within the label - //match_start = w.label.find(r.route) - //if match_start >= 0: - // we have a potential match, just need to make sure if the route - // name ends with a number that the matched substring isn't followed - // by more numbers (e.g., NY50 is an OK label in NY5) - // if len(r.route) + match_start == len(w.label) or \ - // not w.label[len(r.route) + match_start].isdigit(): // partially complete "references own route" -- too many FP - //or re.fullmatch('.*/'+r.route+'.*',w.label[w.label) : // first check for number match after a slash, if there is one if (slash && route->route.back() >= '0' && route->route.back() <= '9') { int digit_starts = route->route.size()-1; diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 820e4817..fc2a0bc1 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -3887,15 +3887,7 @@ def run(self): visible_distance = 0.0 # looking for the route within the label - #match_start = w.label.find(r.route) - #if match_start >= 0: - # we have a potential match, just need to make sure if the route - # name ends with a number that the matched substring isn't followed - # by more numbers (e.g., NY50 is an OK label in NY5) - # if len(r.route) + match_start == len(w.label) or \ - # not w.label[len(r.route) + match_start].isdigit(): # partially complete "references own route" -- too many FP - #or re.fullmatch('.*/'+r.route+'.*',w.label[w.label) : # first check for number match after a slash, if there is one selfref_found = False if '/' in w.label and r.route[-1].isdigit(): @@ -3910,7 +3902,6 @@ def run(self): selfref_found = True if '_' in w.label[w.label.index('/')+1:] and w.label[w.label.index('/')+1:w.label.rindex('_')] == r.route: selfref_found = True - # now the remaining checks if selfref_found or r.route+r.banner == w.label or re.fullmatch(r.route+r.banner+'[_/].*',w.label): datacheckerrors.append(DatacheckEntry(r,[w.label],'LABEL_SELFREF')) From dc37d8ab88c17d32ed95408d1c6ca1615b144042 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Tue, 17 Nov 2020 10:13:32 -0500 Subject: [PATCH 34/34] expand US_LETTER datacheck to account for city abbrevs --- .../cplusplus/classes/Waypoint/Waypoint.cpp | 6 +++++ siteupdate/python-teresco/siteupdate.py | 23 +++++++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp index d1c4ca9e..d1c49127 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp @@ -529,6 +529,12 @@ inline void Waypoint::us_letter(DatacheckEntryList *datacheckerrors) if (*c < 'A' || *c++ > 'B') return; if (*c == 0 || *c == '/' || *c == '_' || *c == '(') datacheckerrors->add(route, label, "", "", "US_LETTER", ""); + // is it followed by a city abbrev? + else if (*c >= 'A' && *c++ <= 'Z' + && *c >= 'a' && *c++ <= 'z' + && *c >= 'a' && *c++ <= 'z' + && *c == 0 || *c == '/' || *c == '_' || *c == '(') + datacheckerrors->add(route, label, "", "", "US_LETTER", ""); } inline void Waypoint::visible_distance(DatacheckEntryList *datacheckerrors, char *fstr, double &vis_dist, Waypoint *&last_visible) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index fc2a0bc1..ce97b38b 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -3953,22 +3953,37 @@ def run(self): # USA-only datachecks if usa_flag and len(w.label) >= 2: + # look for I-xx with Bus instead of BL or BS if re.fullmatch('\*?I\-[0-9]+[EeWwCcNnSs]?[Bb][Uu][Ss].*', w.label): datacheckerrors.append(DatacheckEntry(r,[w.label],'BUS_WITH_I')) + # look for Ixx without hyphen c = 1 if w.label[0] == '*' else 0 if w.label[c:c+2] == "To": c += 2; if len(w.label) >= c+2 and w.label[c] == 'I' and w.label[c+1].isdigit(): datacheckerrors.append(DatacheckEntry(r,[w.label],'INTERSTATE_NO_HYPHEN')) + # look for USxxxA but not USxxxAlt, B/Bus/Byp # Eric's paraphrase of Jim's original criteria # if re.fullmatch('\*?US[0-9]+[AB].*', w.label) and not re.fullmatch('\*?US[0-9]+Alt.*|\*?US[0-9]+Bus.*|\*?US[0-9]+Byp.*', w.label): - # Instead, let's cast a narrower net - if re.fullmatch('\*?US[0-9]+[AB]|\*?US[0-9]+[AB][/_(].*', w.label): - datacheckerrors.append(DatacheckEntry(r,[w.label],'US_LETTER')) - + # Instead, let's cast a narrower net (optimized for speed, just because): + try: + c = 3 if w.label[0] == '*' else 2 + if w.label[c-2] == 'U' and w.label[c-1] == 'S' and w.label[c].isdigit(): + while w.label[c].isdigit(): + c += 1 + if w.label[c] in 'AB': + c += 1 + if c == len(w.label) or w.label[c] in '/_(': + datacheckerrors.append(DatacheckEntry(r,[w.label],'US_LETTER')) + # is it followed by a city abbrev? + elif w.label[c].isupper() and w.label[c+1].islower() and w.label[c+2].islower() \ + and (c+3 == len(w.label) or w.label[c+3] in '/_('): + datacheckerrors.append(DatacheckEntry(r,[w.label],'US_LETTER')) + except IndexError: + pass prev_w = w # angle check is easier with a traditional for loop and array indices