From 079b9ddb578b99534c58cb541cd973f9acc89df0 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Sun, 4 Nov 2018 00:22:38 -0400 Subject: [PATCH 1/7] copyright; plural->singular --- siteupdate/python-teresco/siteupdate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 0f47c36b..ef9aa5e0 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -# Travel Mapping Project, Jim Teresco, 2015, 2016, 2017 +# Travel Mapping Project, Jim Teresco, 2015-2018 """Python code to read .csv and .wpt files and prepare for adding to the Travel Mapping Project database. -(c) 2015, 2016, 2017, Jim Teresco +(c) 2015-2018, Jim Teresco This module defines classes to represent the contents of a .csv file that lists the highways within a system, and a @@ -1035,7 +1035,7 @@ class DatacheckEntry: form a sharp angle) code is the error code string, one of SHARP_ANGLE, BAD_ANGLE, - DUPLICATE_LABELS, DUPLICATE_COORDS, LABEL_SELFREF, + DUPLICATE_LABEL, DUPLICATE_COORDS, LABEL_SELFREF, LABEL_INVALID_CHAR, LONG_SEGMENT, LABEL_NO_VALID, LABEL_UNDERSCORES, VISIBLE_DISTANCE, LABEL_PARENS, LACKS_GENERIC, BUS_WITH_I, NONTERMINAL_UNDERSCORE, From d1729ce029cbae7fd578e3d3f37b9c36c408bfc0 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Sun, 4 Nov 2018 15:12:11 -0500 Subject: [PATCH 2/7] remove LABEL_NO_VALID --- siteupdate/python-teresco/siteupdate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index ef9aa5e0..1f9a3381 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1036,7 +1036,7 @@ class DatacheckEntry: code is the error code string, one of SHARP_ANGLE, BAD_ANGLE, DUPLICATE_LABEL, DUPLICATE_COORDS, LABEL_SELFREF, - LABEL_INVALID_CHAR, LONG_SEGMENT, LABEL_NO_VALID, + LABEL_INVALID_CHAR, LONG_SEGMENT, LABEL_UNDERSCORES, VISIBLE_DISTANCE, LABEL_PARENS, LACKS_GENERIC, BUS_WITH_I, NONTERMINAL_UNDERSCORE, LONG_UNDERSCORE, LABEL_SLASHES, US_BANNER, VISIBLE_HIDDEN_COLOC, From 90329cdf47a420f80bf6cb77595f3e85e6550d17 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Mon, 5 Nov 2018 17:17:51 -0500 Subject: [PATCH 3/7] Quadtree: prevent infinite insert/refine recursion --- siteupdate/python-teresco/siteupdate.py | 34 +++++++++++++++++++------ 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 0f47c36b..9a95766b 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -58,6 +58,7 @@ def __init__(self,min_lat,min_lng,max_lat,max_lng): self.sw_child = None self.se_child = None self.points = [] + self.unique_locations = 0 def refine(self): """refine a quadtree into 4 sub-quadrants""" @@ -71,13 +72,15 @@ def refine(self): for p in points: self.insert(p) - def insert(self,w): """insert Waypoint w into this quadtree node""" #print("QTDEBUG: " + str(self) + " insert " + str(w)) if self.points is not None: + if self.waypoint_at_same_point(w) is None: + self.unique_locations += 1 + #print("QTDEBUG: " + str(self) + " has " + str(self.unique_locations) + " unique locations") self.points.append(w) - if len(self.points) > 50: # 50 points max per quadtree node + if self.unique_locations > 50: # 50 unique points max per quadtree node self.refine() else: if w.lat < self.mid_lat: @@ -195,8 +198,8 @@ def is_valid(self): else: # not refined, but should have no more than 50 points - if len(self.points) > 50: - print("ERROR: WaypointQuadtree.is_valid terminal quadrant has too many points (" + str(len(self.points)) + ")") + if len(self.unique_locations) > 50: + print("ERROR: WaypointQuadtree.is_valid terminal quadrant has too many unique points (" + str(len(self.unique_locations)) + ")") return False # not refined, so should not have any children if self.nw_child is not None: @@ -214,6 +217,22 @@ def is_valid(self): return True + def max_colocated(self): + """return the maximum number of waypoints colocated at any one location""" + max_col = 1 + for p in self.point_list(): + if max_col < p.num_colocated(): + max_col = p.num_colocated() + print("Largest colocate count = " + str(max_col)) + return max_col + + def total_nodes(self): + if self.points is not None: + # not refined, no children, return 1 for self + return 1 + else: + return 1 + self.nw_child.total_nodes() + self.ne_child.total_nodes() + self.sw_child.total_nodes() + self.se_child.total_nodes() + class Waypoint: """This class encapsulates the information about a single waypoint from a .wpt file. @@ -3592,12 +3611,13 @@ def run(self): str(points) + " points and " + str(segments) + " segments.") if points != all_waypoints.size(): print("MISMATCH: all_waypoints contains " + str(all_waypoints.size()) + " waypoints!") +print("WaypointQuadtree contains " + str(all_waypoints.total_nodes()) + " total nodes.") if not args.errorcheck: # compute colocation of waypoints stats print(et.et() + "Computing waypoint colocation stats, reporting all with 8 or more colocations:") - colocate_counts = [0]*50 - largest_colocate_count = 1 + largest_colocate_count = all_waypoints.max_colocated() + colocate_counts = [0]*(largest_colocate_count+1) big_colocate_locations = dict() for w in all_waypoints.point_list(): c = w.num_colocated() @@ -3614,8 +3634,6 @@ def run(self): big_colocate_locations[point] = the_list #print(str(w) + " with " + str(c) + " other points.") colocate_counts[c] += 1 - if c > largest_colocate_count: - largest_colocate_count = c for place in big_colocate_locations: the_list = big_colocate_locations[place] print(str(place) + " is occupied by " + str(len(the_list)) + " waypoints: " + str(the_list)) From 75733a2c51733b9b421ade35b629033127a95b9d Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Mon, 5 Nov 2018 23:14:19 -0500 Subject: [PATCH 4/7] WaypointQuadtree sanity check fix --- siteupdate/python-teresco/siteupdate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 9a95766b..704885e6 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -198,8 +198,8 @@ def is_valid(self): else: # not refined, but should have no more than 50 points - if len(self.unique_locations) > 50: - print("ERROR: WaypointQuadtree.is_valid terminal quadrant has too many unique points (" + str(len(self.unique_locations)) + ")") + if self.unique_locations > 50: + print("ERROR: WaypointQuadtree.is_valid terminal quadrant has too many unique points (" + str(self.unique_locations) + ")") return False # not refined, so should not have any children if self.nw_child is not None: From 316261db3e2037e5dc67fa41478840bec348b1ac Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Tue, 6 Nov 2018 00:25:47 -0500 Subject: [PATCH 5/7] nmpbyregion.log & nmpbycountry.log --- nmpfilter/nmpbycountry.cpp | 24 ++++++++++++++++++++++++ nmpfilter/nmpbyregion.cpp | 23 +++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/nmpfilter/nmpbycountry.cpp b/nmpfilter/nmpbycountry.cpp index 09290a28..b1a125fc 100644 --- a/nmpfilter/nmpbycountry.cpp +++ b/nmpfilter/nmpbycountry.cpp @@ -1,4 +1,5 @@ // Travel Mapping Project, Eric Bryant, 2018 +#include #include #include #include @@ -100,6 +101,29 @@ int main(int argc, char *argv[]) { cout << "usage: nmpbycountry RgCsvFile MasterNMP OutputDir\n"; return 0; } + + // Record execution start time + time_t StartTime = time(0); + char* LocalTime = ctime(&StartTime); + + // Attempt to find most recent commit info + string MasterInfo; + string MasterPath = argv[1]; + MasterPath.erase(MasterPath.find_last_of("/\\")+1); + MasterPath += ".git/refs/heads/master"; + ifstream MasterFile(MasterPath.data()); + if (MasterFile) + MasterFile >> MasterInfo; + else MasterInfo = "unknown."; + + // nmpbyregion.log + string LogPath = argv[3]; + LogPath += "nmpbycountry.log"; + ofstream LogFile(LogPath.data()); + LogFile << "nmpbycountry executed " << LocalTime; + LogFile << "Most recent commit is " << MasterInfo << '\n'; + + // The actual filtering list CoList; vector RgList; vector master; diff --git a/nmpfilter/nmpbyregion.cpp b/nmpfilter/nmpbyregion.cpp index 64481b55..cd3dcaaf 100644 --- a/nmpfilter/nmpbyregion.cpp +++ b/nmpfilter/nmpbyregion.cpp @@ -1,4 +1,5 @@ // Travel Mapping Project, Eric Bryant, 2018 +#include #include #include #include @@ -63,6 +64,28 @@ int main(int argc, char *argv[]) { cout << "usage: nmpbyregion HwyDataDir MasterNMP OutputDir\n"; return 0; } + + // Record execution start time + time_t StartTime = time(0); + char* LocalTime = ctime(&StartTime); + + // Attempt to find most recent commit info + string MasterInfo; + string MasterPath = argv[1]; + MasterPath += "../.git/refs/heads/master"; + ifstream MasterFile(MasterPath.data()); + if (MasterFile) + MasterFile >> MasterInfo; + else MasterInfo = "unknown."; + + // nmpbyregion.log + string LogPath = argv[3]; + LogPath += "nmpbyregion.log"; + ofstream LogFile(LogPath.data()); + LogFile << "nmpbyregion executed " << LocalTime; + LogFile << "Most recent commit is " << MasterInfo << '\n'; + + // The actual filtering vector RgList; GetRegions(argv[1], RgList); filter(RgList, argv[2], argv[3]); From fbdec000aacb5c9b15df5dfde11d5c87bccde4fe Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Tue, 6 Nov 2018 00:57:19 -0500 Subject: [PATCH 6/7] nmpbycountry requires c++11 --- nmpfilter/README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/nmpfilter/README.md b/nmpfilter/README.md index ae670548..f9edd634 100644 --- a/nmpfilter/README.md +++ b/nmpfilter/README.md @@ -3,7 +3,11 @@ **Purpose:**
Splits *tm-master.nmp* into smaller .nmp files filtered by country. -**usage:**
+**Compiling:**
+C++11 support is required.
+With GCC, I use the commandline `g++ nmpbycountry.cpp -std=c++11 -o nmpbycountry` + +**Usage:**
`nmpbycountry RgCsvFile MasterNMP OutputDir`, where * `RgCsvFile` is the path of *regions.csv* * `MasterNMP` is the path of *tm-master.nmp* @@ -18,7 +22,7 @@ The output directory should be empty before running *nmpbycountry*. Files for co **Purpose:**
Splits *tm-master.nmp* into smaller .nmp files filtered by region. -**usage:**
+**Usage:**
`nmpbyregion HwyDataDir MasterNMP OutputDir`, where * `HwyDataDir` is the path of the *hwy_data* directory in the HighwayData repository, or equivalent * `MasterNMP` is the path of *tm-master.nmp* @@ -31,7 +35,7 @@ Splits *tm-master.nmp* into smaller .nmp files filtered by region. **Purpose:**
Removes marked false-positive pairs from a specified .nmp file. -**usage:**
+**Usage:**
`nmpfpfilter InputFile OutputFile` --- @@ -41,5 +45,5 @@ Removes marked false-positive pairs from a specified .nmp file. **Purpose:**
Removes all NMP pairs from a specified .nmp file except those that look intentional. -**usage:**
+**Usage:**
`nmplifilter InputFile OutputFile` From bffdbf9fd80be3d02260e761477efad9d63195b6 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Tue, 6 Nov 2018 02:15:43 -0500 Subject: [PATCH 7/7] more compact, readable, robust & sensible fpcull * replace input stream extractors with getline * use std::list containers instead of custom classes * basically rewrite the whole thing --- fpcull/fpcull.cpp | 67 ++++++++++------------------------------------- 1 file changed, 14 insertions(+), 53 deletions(-) diff --git a/fpcull/fpcull.cpp b/fpcull/fpcull.cpp index eb070c3f..685076e2 100644 --- a/fpcull/fpcull.cpp +++ b/fpcull/fpcull.cpp @@ -1,36 +1,11 @@ // Travel Mapping Project, Eric Bryant, 2017, 2018 #include #include +#include using namespace std; -class entry -{ public: - string text; - bool cull; - entry *prev, *next; - - // init alpha - entry() {prev = 0; next = 0; cull = 0;} - // insertions - entry(entry *cursor) // cursor, in this case, is a pointer to the prev entry - { prev = cursor; - next = 0; - cull = 0; - } -}; - -bool find(string needle, entry *haystack) -{ while (haystack) - { if (needle == haystack->text) return 1; - haystack = haystack->next; - } - return 0; -} - int main(int argc, char *argv[]) -{ entry *cList = new entry; - entry *fList = new entry; - entry *cursor; +{ string FPline; if (argc != 4) { cout << "usage: ./fpcull CullFile FullFile OutputFile\n"; return 0; } // init ifstreams ifstream cFile(argv[1], ios::in); @@ -38,35 +13,21 @@ int main(int argc, char *argv[]) ifstream fFile(argv[2], ios::in); if (!fFile.is_open()) { cout << argv[2] << " file not found!\n"; return 0; } - // create cull list - cFile >> cList->text; //TODO: enclose in an IF in case hell breaks loose (IE, no text in cFile) //FIXME: see below - cursor = new entry(cList); - while (cFile >> cursor->text) //FIXME: This expects every line to be one text string and every text string to be one line. This will not always be the case. - // For example, see line 6613 of https://github.com/TravelMapping/HighwayData/blob/d33d0f6e93eec8ce7f71f31af94336a473161e5f/datacheckfps.csv - { cursor->prev->next = cursor; - cursor = new entry(cursor); - } - delete cursor; - // create full list - fFile >> fList->text; //TODO: enclose in an IF in case hell breaks loose (IE, no text in fFile) //FIXME: see below - cursor = new entry(fList); - while (fFile >> cursor->text) //FIXME: This expects every line to be one text string and every text string to be one line. This will not always be the case. - // For example, see line 6613 of https://github.com/TravelMapping/HighwayData/blob/d33d0f6e93eec8ce7f71f31af94336a473161e5f/datacheckfps.csv - { cursor->prev->next = cursor; - cursor = new entry(cursor); + list fList; + while (getline(fFile, FPline)) fList.push_back(FPline); + + // read cull list... + while (getline(cFile, FPline)) + { list::iterator cursor = fList.begin(); + // advance thru full FP list until current unmatched entry found... + while (*cursor != FPline && cursor != fList.end()) cursor++; + // ...and delete it. + if (cursor != fList.end()) cursor = fList.erase(cursor); } - delete cursor; - - // flag unmatchedfps as to-be-culled - for (cursor = fList; cursor; cursor = cursor->next) - if (find(cursor->text, cList)) cursor->cull = 1; // output ofstream oFile(argv[3]); - cursor = fList; - while (cursor) - { if (!cursor->cull) oFile << cursor->text << endl; - cursor = cursor->next; - } + for (list::iterator cursor = fList.begin(); cursor != fList.end(); cursor++) + oFile << *cursor << '\n'; }