From 2c75863fe6dafd6347a11c2e1fcbd17920cebe80 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Sat, 25 Apr 2020 02:36:25 -0400 Subject: [PATCH 1/5] wpt whitespace/blank line bugfix --- siteupdate/cplusplus/classes/Route/read_wpt.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteupdate/cplusplus/classes/Route/read_wpt.cpp b/siteupdate/cplusplus/classes/Route/read_wpt.cpp index 1be5cadc..106b3dd0 100644 --- a/siteupdate/cplusplus/classes/Route/read_wpt.cpp +++ b/siteupdate/cplusplus/classes/Route/read_wpt.cpp @@ -39,7 +39,7 @@ void Route::read_wpt { // strip whitespace while (lines[l][0] == ' ' || lines[l][0] == '\t') lines[l]++; char * endchar = lines[l+1]-2; // -2 skips over the 0 inserted by strtok - if (*endchar == 0) endchar--; // skip back one more for CRLF cases FIXME what about lines followed by blank lines? + while (*endchar == 0) endchar--; // skip back more for CRLF cases, and lines followed by blank lines while (*endchar == ' ' || *endchar == '\t') { *endchar = 0; endchar--; From 14ea929383adef4b52e2d3e4d1666b369ee180cc Mon Sep 17 00:00:00 2001 From: eric bryant Date: Sat, 25 Apr 2020 02:57:29 -0400 Subject: [PATCH 2/5] no strtok_mtx when reading waypoints --- siteupdate/cplusplus/classes/Route/Route.h | 2 +- .../cplusplus/classes/Route/read_wpt.cpp | 19 ++++++++++++++----- .../cplusplus/classes/Waypoint/Waypoint.cpp | 15 ++++++++++----- .../cplusplus/classes/Waypoint/Waypoint.h | 2 +- siteupdate/cplusplus/siteupdate.cpp | 4 ++-- .../cplusplus/threads/ReadWptThread.cpp | 4 ++-- 6 files changed, 30 insertions(+), 16 deletions(-) diff --git a/siteupdate/cplusplus/classes/Route/Route.h b/siteupdate/cplusplus/classes/Route/Route.h index b1949da7..c67a1479 100644 --- a/siteupdate/cplusplus/classes/Route/Route.h +++ b/siteupdate/cplusplus/classes/Route/Route.h @@ -61,7 +61,7 @@ class Route Route(std::string &, HighwaySystem *, ErrorList &, std::unordered_map &); std::string str(); - void read_wpt(WaypointQuadtree *, ErrorList *, std::string, std::mutex *, DatacheckEntryList *, std::unordered_set *); + void read_wpt(WaypointQuadtree *, ErrorList *, std::string, DatacheckEntryList *, std::unordered_set *); void print_route(); HighwaySegment* find_segment_by_waypoints(Waypoint*, Waypoint*); std::string chopped_rtes_line(); diff --git a/siteupdate/cplusplus/classes/Route/read_wpt.cpp b/siteupdate/cplusplus/classes/Route/read_wpt.cpp index 106b3dd0..bc261744 100644 --- a/siteupdate/cplusplus/classes/Route/read_wpt.cpp +++ b/siteupdate/cplusplus/classes/Route/read_wpt.cpp @@ -1,5 +1,5 @@ void Route::read_wpt -( WaypointQuadtree *all_waypoints, ErrorList *el, std::string path, std::mutex *strtok_mtx, +( WaypointQuadtree *all_waypoints, ErrorList *el, std::string path, DatacheckEntryList *datacheckerrors, std::unordered_set *all_wpt_files ) { /* read data into the Route's waypoint list from a .wpt file */ @@ -23,9 +23,18 @@ void Route::read_wpt file.read(wptdata, wptdatasize); wptdata[wptdatasize] = 0; // add null terminator file.close(); - strtok_mtx->lock(); - for (char *token = strtok(wptdata, "\r\n"); token; token = strtok(0, "\r\n") ) lines.emplace_back(token); - strtok_mtx->unlock(); + + // split file into lines + size_t spn = 0; + for (char* c = wptdata; *c; c += spn) + { spn = strcspn(c, "\n\r"); + while (c[spn] == '\n' || c[spn] == '\r') + { c[spn] = 0; + spn++; + } + lines.emplace_back(c); + } + lines.push_back(wptdata+wptdatasize+1); // add a dummy "past-the-end" element to make lines[l+1]-2 work // set to be used per-route to find label duplicates std::unordered_set all_route_labels; @@ -45,7 +54,7 @@ void Route::read_wpt endchar--; } if (lines[l][0] == 0) continue; - Waypoint *w = new Waypoint(lines[l], this, strtok_mtx, datacheckerrors); + Waypoint *w = new Waypoint(lines[l], this, datacheckerrors); // deleted on termination of program, or immediately below if invalid bool malformed_url = w->lat == 0 && w->lng == 0; bool label_too_long = w->label_too_long(datacheckerrors); diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp index 890724bf..7b3bfe20 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp @@ -15,15 +15,20 @@ bool waypoint_simplification_sort(Waypoint *w1, Waypoint *w2) const double Waypoint::pi = 3.141592653589793238; -Waypoint::Waypoint(char *line, Route *rte, std::mutex *strtok_mtx, DatacheckEntryList *datacheckerrors) +Waypoint::Waypoint(char *line, Route *rte, DatacheckEntryList *datacheckerrors) { /* initialize object from a .wpt file line */ route = rte; // parse WPT line - strtok_mtx->lock(); - for (char *token = strtok(line, " "); token; token = strtok(0, " ")) - alt_labels.emplace_back(token); // get all tokens & put into label deque - strtok_mtx->unlock(); + size_t spn = 0; + for (char* c = line; *c; c += spn) + { spn = strcspn(c, " "); + while (c[spn] == ' ') + { c[spn] = 0; + spn++; + } + alt_labels.emplace_back(c); + } // We know alt_labels will have at least one element, because if the WPT line is // blank or contains only spaces, Route::read_wpt will not call this constructor. diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.h b/siteupdate/cplusplus/classes/Waypoint/Waypoint.h index 694c2c90..a419e8d3 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.h +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.h @@ -22,7 +22,7 @@ class Waypoint bool is_hidden; static const double pi; - Waypoint(char *, Route *, std::mutex *, DatacheckEntryList *); + Waypoint(char *, Route *, DatacheckEntryList *); std::string str(); std::string csv_line(unsigned int); diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index 90f97f55..cef22302 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -335,7 +335,7 @@ int main(int argc, char *argv[]) thread **thr = new thread*[args.numthreads]; for (unsigned int t = 0; t < args.numthreads; t++) thr[t] = new thread(ReadWptThread, t, &highway_systems, &hs_it, &list_mtx, args.highwaydatapath+"/hwy_data", - &el, &all_wpt_files, &all_waypoints, &strtok_mtx, datacheckerrors); + &el, &all_wpt_files, &all_waypoints, datacheckerrors); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); for (unsigned int t = 0; t < args.numthreads; t++) @@ -344,7 +344,7 @@ int main(int argc, char *argv[]) for (HighwaySystem* h : highway_systems) { std::cout << h->systemname << std::flush; for (std::list::iterator r = h->route_list.begin(); r != h->route_list.end(); r++) - { r->read_wpt(&all_waypoints, &el, args.highwaydatapath+"/hwy_data", &strtok_mtx, datacheckerrors, &all_wpt_files); + { r->read_wpt(&all_waypoints, &el, args.highwaydatapath+"/hwy_data", datacheckerrors, &all_wpt_files); } std::cout << "!" << std::endl; } diff --git a/siteupdate/cplusplus/threads/ReadWptThread.cpp b/siteupdate/cplusplus/threads/ReadWptThread.cpp index e4c00de8..712111f7 100644 --- a/siteupdate/cplusplus/threads/ReadWptThread.cpp +++ b/siteupdate/cplusplus/threads/ReadWptThread.cpp @@ -1,7 +1,7 @@ void ReadWptThread ( unsigned int id, std::list *hs_list, std::list::iterator *it, std::mutex *hs_mtx, std::string path, ErrorList *el, std::unordered_set *all_wpt_files, - WaypointQuadtree *all_waypoints, std::mutex *strtok_mtx, DatacheckEntryList *datacheckerrors + WaypointQuadtree *all_waypoints, DatacheckEntryList *datacheckerrors ) { //printf("Starting ReadWptThread %02i\n", id); fflush(stdout); while (*it != hs_list->end()) @@ -17,7 +17,7 @@ void ReadWptThread hs_mtx->unlock(); std::cout << h->systemname << std::flush; for (Route &r : h->route_list) - r.read_wpt(all_waypoints, el, path, strtok_mtx, datacheckerrors, all_wpt_files); + r.read_wpt(all_waypoints, el, path, datacheckerrors, all_wpt_files); std::cout << "!" << std::endl; } } From 34c407ac011a9f54c28499390d4cdab28f62fbad Mon Sep 17 00:00:00 2001 From: eric bryant Date: Sun, 26 Apr 2020 13:16:25 -0400 Subject: [PATCH 3/5] update comment: no more strtok --- siteupdate/cplusplus/classes/Route/read_wpt.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteupdate/cplusplus/classes/Route/read_wpt.cpp b/siteupdate/cplusplus/classes/Route/read_wpt.cpp index bc261744..433c025f 100644 --- a/siteupdate/cplusplus/classes/Route/read_wpt.cpp +++ b/siteupdate/cplusplus/classes/Route/read_wpt.cpp @@ -47,7 +47,7 @@ void Route::read_wpt for (unsigned int l = 0; l < lines.size()-1; l++) { // strip whitespace while (lines[l][0] == ' ' || lines[l][0] == '\t') lines[l]++; - char * endchar = lines[l+1]-2; // -2 skips over the 0 inserted by strtok + char * endchar = lines[l+1]-2; // -2 skips over the 0 inserted while splitting wptdata into lines while (*endchar == 0) endchar--; // skip back more for CRLF cases, and lines followed by blank lines while (*endchar == ' ' || *endchar == '\t') { *endchar = 0; From e0c21d110b1a4b8d556d935ac9a31f9552309025 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Mon, 27 Apr 2020 11:02:16 -0400 Subject: [PATCH 4/5] all_wpt_files: list -> set --- siteupdate/python-teresco/siteupdate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index d2cce465..5cb4cdb1 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2493,11 +2493,11 @@ def __init__(self,filename,descr,vertices,edges,travelers,format,category): # that do not have a .csv file entry that causes them to be # read into the data print(et.et() + "Finding all .wpt files. ",end="",flush=True) -all_wpt_files = [] +all_wpt_files = set() for dir, sub, files in os.walk(args.highwaydatapath+"/hwy_data"): for file in files: if file.endswith('.wpt') and '_boundaries' not in dir: - all_wpt_files.append(dir+"/"+file) + all_wpt_files.add(dir+"/"+file) print(str(len(all_wpt_files)) + " files found.") # For finding colocated Waypoints and concurrent segments, we have From 8c633d2dff6bb430bc79e4f84dd4cdbce986e3f5 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Mon, 27 Apr 2020 14:12:44 -0400 Subject: [PATCH 5/5] syntactic sugar --- siteupdate/python-teresco/siteupdate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 5cb4cdb1..9e764491 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -3600,7 +3600,7 @@ def run(self): if r.point_list[0].is_hidden: datacheckerrors.append(DatacheckEntry(r,[r.point_list[0].label],'HIDDEN_TERMINUS')) if r.point_list[-1].is_hidden: - datacheckerrors.append(DatacheckEntry(r,[r.point_list[len(r.point_list)-1].label],'HIDDEN_TERMINUS')) + datacheckerrors.append(DatacheckEntry(r,[r.point_list[-1].label],'HIDDEN_TERMINUS')) for w in r.point_list: # duplicate labels