From abd17c8a9d61a86714cfdd58555c9b3607c9d1b0 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Thu, 26 Mar 2020 17:17:59 -0400 Subject: [PATCH 1/3] return 1 with fatal errors --- siteupdate/cplusplus/siteupdate.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index 29e022c0..941f7ca8 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -121,9 +121,7 @@ int main(int argc, char *argv[]) } closedir(dir); } - else { cout << "Error opening directory " << args.userlistfilepath << ". (Not found?)" << endl; - return 0; - } + else el.add_error("Error opening user list file path \""+args.userlistfilepath+"\". (Not found?)"); } else for (string &id : traveler_ids) id += ".list"; @@ -952,7 +950,7 @@ int main(int argc, char *argv[]) { cout << "ABORTING due to " << el.error_list.size() << " errors:" << endl; for (unsigned int i = 0; i < el.error_list.size(); i++) cout << i+1 << ": " << el.error_list[i] << endl; - return 0; + return 1; } #ifdef threading_enabled From dd2c4d7857061ad2244d93ec88018ffea9fdafda Mon Sep 17 00:00:00 2001 From: eric bryant Date: Fri, 27 Mar 2020 14:20:06 -0400 Subject: [PATCH 2/3] descriptive DatacheckEntry info comments --- .../cplusplus/classes/DatacheckEntry.cpp | 61 +++++++++---------- siteupdate/python-teresco/siteupdate.py | 61 +++++++++---------- 2 files changed, 58 insertions(+), 64 deletions(-) diff --git a/siteupdate/cplusplus/classes/DatacheckEntry.cpp b/siteupdate/cplusplus/classes/DatacheckEntry.cpp index 57f101da..b2ef9686 100644 --- a/siteupdate/cplusplus/classes/DatacheckEntry.cpp +++ b/siteupdate/cplusplus/classes/DatacheckEntry.cpp @@ -7,38 +7,35 @@ class DatacheckEntry as the endpoints of a too-long segment or the three points that form a sharp angle) - code is the error code string, one of: - BAD_ANGLE - BUS_WITH_I - DUPLICATE_COORDS - DUPLICATE_LABEL - HIDDEN_JUNCTION - HIDDEN_TERMINUS - INVALID_FINAL_CHAR - INVALID_FIRST_CHAR - LABEL_INVALID_CHAR - LABEL_LOOKS_HIDDEN - LABEL_PARENS - LABEL_SELFREF - LABEL_SLASHES - LABEL_UNDERSCORES - LACKS_GENERIC - LONG_SEGMENT - LONG_UNDERSCORE - MALFORMED_LAT - MALFORMED_LON - MALFORMED_URL - NONTERMINAL_UNDERSCORE - OUT_OF_BOUNDS - SHARP_ANGLE - US_BANNER - VISIBLE_DISTANCE - VISIBLE_HIDDEN_COLOC - - info is additional information, at this time either a distance (in - miles) for a long segment error, an angle (in degrees) for a sharp - angle error, or a coordinate pair for duplicate coordinates, other - route/label for point pair errors + code is the error code | info is additional + string, one of: | information, if used: + -----------------------+-------------------------------------------- + BAD_ANGLE | + BUS_WITH_I | + DUPLICATE_COORDS | coordinate pair + DUPLICATE_LABEL | + HIDDEN_JUNCTION | number of incident edges in TM master graph + HIDDEN_TERMINUS | + INVALID_FINAL_CHAR | final character in label + INVALID_FIRST_CHAR | first character in label other than * + LABEL_INVALID_CHAR | + LABEL_LOOKS_HIDDEN | + LABEL_PARENS | + LABEL_SELFREF | + LABEL_SLASHES | + LABEL_UNDERSCORES | + LACKS_GENERIC | + LONG_SEGMENT | distance in miles + LONG_UNDERSCORE | + MALFORMED_LAT | malformed "lat=" parameter from OSM url + MALFORMED_LON | malformed "lon=" parameter from OSM url + MALFORMED_URL | always "MISSING_ARG(S)" + NONTERMINAL_UNDERSCORE | + OUT_OF_BOUNDS | coordinate pair + SHARP_ANGLE | angle in degrees + US_BANNER | + VISIBLE_DISTANCE | distance in miles + VISIBLE_HIDDEN_COLOC | hidden point at same coordinates fp is a boolean indicating whether this has been reported as a false positive (would be set to true later) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 59b2bda8..9efd85c3 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1249,38 +1249,35 @@ class DatacheckEntry: as the endpoints of a too-long segment or the three points that form a sharp angle) - code is the error code string, one of: - BAD_ANGLE - BUS_WITH_I - DUPLICATE_COORDS - DUPLICATE_LABEL - HIDDEN_JUNCTION - HIDDEN_TERMINUS - INVALID_FINAL_CHAR - INVALID_FIRST_CHAR - LABEL_INVALID_CHAR - LABEL_LOOKS_HIDDEN - LABEL_PARENS - LABEL_SELFREF - LABEL_SLASHES - LABEL_UNDERSCORES - LACKS_GENERIC - LONG_SEGMENT - LONG_UNDERSCORE - MALFORMED_LAT - MALFORMED_LON - MALFORMED_URL - NONTERMINAL_UNDERSCORE - OUT_OF_BOUNDS - SHARP_ANGLE - US_BANNER - VISIBLE_DISTANCE - VISIBLE_HIDDEN_COLOC - - info is additional information, at this time either a distance (in - miles) for a long segment error, an angle (in degrees) for a sharp - angle error, or a coordinate pair for duplicate coordinates, other - route/label for point pair errors + code is the error code | info is additional + string, one of: | information, if used: + -----------------------+-------------------------------------------- + BAD_ANGLE | + BUS_WITH_I | + DUPLICATE_COORDS | coordinate pair + DUPLICATE_LABEL | + HIDDEN_JUNCTION | number of incident edges in TM master graph + HIDDEN_TERMINUS | + INVALID_FINAL_CHAR | final character in label + INVALID_FIRST_CHAR | first character in label other than * + LABEL_INVALID_CHAR | + LABEL_LOOKS_HIDDEN | + LABEL_PARENS | + LABEL_SELFREF | + LABEL_SLASHES | + LABEL_UNDERSCORES | + LACKS_GENERIC | + LONG_SEGMENT | distance in miles + LONG_UNDERSCORE | + MALFORMED_LAT | malformed "lat=" parameter from OSM url + MALFORMED_LON | malformed "lon=" parameter from OSM url + MALFORMED_URL | always "MISSING_ARG(S)" + NONTERMINAL_UNDERSCORE | + OUT_OF_BOUNDS | coordinate pair + SHARP_ANGLE | angle in degrees + US_BANNER | + VISIBLE_DISTANCE | distance in miles + VISIBLE_HIDDEN_COLOC | hidden point at same coordinates fp is a boolean indicating whether this has been reported as a false positive (would be set to true later) From 166bc2c08340557c30d8bd92589fcd3ad4e370dd Mon Sep 17 00:00:00 2001 From: eric bryant Date: Sat, 28 Mar 2020 01:49:01 -0400 Subject: [PATCH 3/3] NMP "looks intentional" fixes --- .../cplusplus/classes/Waypoint/Waypoint.cpp | 70 +++++++------- .../cplusplus/classes/Waypoint/Waypoint.h | 2 +- siteupdate/cplusplus/siteupdate.cpp | 9 +- siteupdate/python-teresco/siteupdate.py | 91 ++++++++++++------- 4 files changed, 104 insertions(+), 68 deletions(-) diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp index c0df7a9d..1a53ce70 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp @@ -249,66 +249,72 @@ std::string Waypoint::root_at_label() { return route->root + "@" + label; } -void Waypoint::nmplogs(std::list &nmpfplist, std::ofstream &nmpnmp, std::list &nmploglines) +void Waypoint::nmplogs(std::unordered_set &nmpfps, std::ofstream &nmpnmp, std::list &nmploglines) { if (!near_miss_points.empty()) { // sort the near miss points for consistent ordering to facilitate NMP FP marking near_miss_points.sort(sort_root_at_label); - bool nmplooksintentional = 0; + // construct string for nearmisspoints.log & FP matching std::string nmpline = str() + " NMP"; - std::list nmpnmplines; + for (Waypoint *other_w : near_miss_points) nmpline += " " + other_w->str(); + // check for string in fp list + std::unordered_set::iterator fpit = nmpfps.find(nmpline); + if (fpit == nmpfps.end()) fpit = nmpfps.find(nmpline+" [LOOKS INTENTIONAL]"); + if (fpit == nmpfps.end()) fpit = nmpfps.find(nmpline+" [SOME LOOK INTENTIONAL]"); + bool fp = fpit != nmpfps.end(); + // write lines to tm-master.nmp + size_t li_count = 0; for (Waypoint *other_w : near_miss_points) - { if ((fabs(lat - other_w->lat) < 0.0000015) && (fabs(lng - other_w->lng) < 0.0000015)) - nmplooksintentional = 1; - nmpline += " " + other_w->str(); + { bool li = (fabs(lat - other_w->lat) < 0.0000015) && (fabs(lng - other_w->lng) < 0.0000015); + if (li) li_count++; // make sure we only plot once, since the NMP should be listed // both ways (other_w in w's list, w in other_w's list) if (sort_root_at_label(this, other_w)) { char coordstr[51]; - std::string nmpnmpline = root_at_label(); + nmpnmp << root_at_label(); sprintf(coordstr, " %.15g", lat); if (!strchr(coordstr, '.')) strcat(coordstr, ".0"); // add single trailing zero to ints for compatibility with Python - nmpnmpline += coordstr; + nmpnmp << coordstr; sprintf(coordstr, " %.15g", lng); if (!strchr(coordstr, '.')) strcat(coordstr, ".0"); // add single trailing zero to ints for compatibility with Python - nmpnmpline += coordstr; - nmpnmplines.push_back(nmpnmpline); - - nmpnmpline = other_w->root_at_label(); + nmpnmp << coordstr; + if (fp || li) + { nmpnmp << ' '; + if (fp) nmpnmp << "FP"; + if (li) nmpnmp << "LI"; + } + nmpnmp << '\n'; + + nmpnmp << other_w->root_at_label(); sprintf(coordstr, " %.15g", other_w->lat); if (!strchr(coordstr, '.')) strcat(coordstr, ".0"); // add single trailing zero to ints for compatibility with Python - nmpnmpline += coordstr; + nmpnmp << coordstr; sprintf(coordstr, " %.15g", other_w->lng); if (!strchr(coordstr, '.')) strcat(coordstr, ".0"); // add single trailing zero to ints for compatibility with Python - nmpnmpline += coordstr; - nmpnmplines.push_back(nmpnmpline); + nmpnmp << coordstr; + if (fp || li) + { nmpnmp << ' '; + if (fp) nmpnmp << "FP"; + if (li) nmpnmp << "LI"; + } + nmpnmp << '\n'; } } // indicate if this was in the FP list or if it's off by exact amt // so looks like it's intentional, and detach near_miss_points list // so it doesn't get a rewrite in nmp_merged WPT files - // also set the extra field to mark FP/LI items in the .nmp file - std::string extra_field; - std::list::iterator fp = nmpfplist.begin(); - while (fp != nmpfplist.end() && *fp != nmpline) fp++; - if (fp != nmpfplist.end()) - { nmpfplist.erase(fp); - nmpline += " [MARKED FP]"; + if (li_count) + { if ( li_count == std::distance(near_miss_points.begin(), near_miss_points.end()) ) + nmpline += " [LOOKS INTENTIONAL]"; + else nmpline += " [SOME LOOK INTENTIONAL]"; near_miss_points.clear(); - extra_field += "FP"; } - if (nmplooksintentional) - { nmpline += " [LOOKS INTENTIONAL]"; + if (fp) + { nmpfps.erase(fpit); + nmpline += " [MARKED FP]"; near_miss_points.clear(); - extra_field += "LI"; } - if (extra_field != "") extra_field = " " + extra_field; nmploglines.push_back(nmpline); - - // write actual lines to .nmp file, indicating FP and/or LI - // for marked FPs or looks intentional items - for (std::string nmpnmpline : nmpnmplines) - nmpnmp << nmpnmpline << extra_field << '\n'; } } diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.h b/siteupdate/cplusplus/classes/Waypoint/Waypoint.h index 72097c1c..18dad668 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.h +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.h @@ -35,7 +35,7 @@ class Waypoint std::string simple_waypoint_name(); bool is_or_colocated_with_active_or_preview(); std::string root_at_label(); - void nmplogs(std::list &, std::ofstream &, std::list &); + void nmplogs(std::unordered_set &, std::ofstream &, std::list &); inline Waypoint* hashpoint(); bool label_references_route(Route *, DatacheckEntryList *); diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index 29e022c0..b2543232 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -356,18 +356,18 @@ int main(int argc, char *argv[]) cout << et.et() << "Near-miss point log and tm-master.nmp file." << endl; // read in fp file - list nmpfplist; + unordered_set nmpfps; file.open(args.highwaydatapath+"/nmpfps.log"); while (getline(file, line)) { while (line.back() == 0x0D || line.back() == ' ') line.erase(line.end()-1); // trim DOS newlines & whitespace - if (line.size()) nmpfplist.push_back(line); + if (line.size()) nmpfps.insert(line); } file.close(); list nmploglines; ofstream nmplog(args.logfilepath+"/nearmisspoints.log"); ofstream nmpnmp(args.logfilepath+"/tm-master.nmp"); - for (Waypoint *w : all_waypoints.point_list()) w->nmplogs(nmpfplist, nmpnmp, nmploglines); + for (Waypoint *w : all_waypoints.point_list()) w->nmplogs(nmpfps, nmpnmp, nmploglines); nmpnmp.close(); // sort and write actual lines to nearmisspoints.log @@ -378,10 +378,13 @@ int main(int argc, char *argv[]) // report any unmatched nmpfps.log entries ofstream nmpfpsunmatchedfile(args.logfilepath+"/nmpfpsunmatched.log"); + list nmpfplist(nmpfps.begin(), nmpfps.end()); + nmpfplist.sort(); for (string &line : nmpfplist) nmpfpsunmatchedfile << line << '\n'; nmpfpsunmatchedfile.close(); nmpfplist.clear(); + nmpfps.clear(); // if requested, rewrite data with near-miss points merged in if (args.nmpmergepath != "" && !args.errorcheck) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 59b2bda8..0942b950 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2414,12 +2414,12 @@ def run(self): print(et.et() + "Near-miss point log and tm-master.nmp file.", flush=True) # read in fp file -nmpfplist = [] +nmpfps = set() nmpfpfile = open(args.highwaydatapath+'/nmpfps.log','r') nmpfpfilelines = nmpfpfile.readlines() for line in nmpfpfilelines: if len(line.rstrip('\n ')) > 0: - nmpfplist.append(line.rstrip('\n ')) + nmpfps.add(line.rstrip('\n ')) nmpfpfile.close() nmploglines = [] @@ -2427,47 +2427,73 @@ def run(self): nmpnmp = open(args.logfilepath+'/tm-master.nmp','w') for w in all_waypoints.point_list(): if w.near_miss_points is not None: - nmpline = str(w) + " NMP " - nmplooksintentional = False - nmpnmplines = [] # sort the near miss points for consistent ordering to facilitate # NMP FP marking - for other_w in sorted(w.near_miss_points, - key=lambda waypoint: - waypoint.route.root + "@" + waypoint.label): - if (abs(w.lat - other_w.lat) < 0.0000015) and \ - (abs(w.lng - other_w.lng) < 0.0000015): - nmplooksintentional = True - nmpline += str(other_w) + " " + w.near_miss_points.sort(key=lambda waypoint: + waypoint.route.root + "@" + waypoint.label) + # construct string for nearmisspoints.log & FP matching + nmpline = str(w) + " NMP" + for other_w in w.near_miss_points: + nmpline += " " + str(other_w) + # check for string in fp list + fp = nmpline in nmpfps + lifp_tag = 0 + if not fp: + if nmpline+" [LOOKS INTENTIONAL]" in nmpfps: + fp = True + lifp_tag = 1 + if not fp: + if nmpline+" [SOME LOOK INTENTIONAL]" in nmpfps: + fp = True + lifp_tag = 2 + # write lines to tm-master.nmp + li_count = 0 + for other_w in w.near_miss_points: + li = (abs(w.lat - other_w.lat) < 0.0000015) and \ + (abs(w.lng - other_w.lng) < 0.0000015) + if li: + li_count += 1 w_label = w.route.root + "@" + w.label other_label = other_w.route.root + "@" + other_w.label # make sure we only plot once, since the NMP should be listed # both ways (other_w in w's list, w in other_w's list) if w_label < other_label: - nmpnmplines.append(w_label + " " + str(w.lat) + " " + str(w.lng)) - nmpnmplines.append(other_label + " " + str(other_w.lat) + " " + str(other_w.lng)) + nmpnmp.write(w_label + " " + str(w.lat) + " " + str(w.lng)) + if fp or li: + nmpnmp.write(' ') + if fp: + nmpnmp.write('FP') + if li: + nmpnmp.write('LI') + nmpnmp.write('\n') + nmpnmp.write(other_label + " " + str(other_w.lat) + " " + str(other_w.lng)) + if fp or li: + nmpnmp.write(' ') + if fp: + nmpnmp.write('FP') + if li: + nmpnmp.write('LI') + nmpnmp.write('\n') # indicate if this was in the FP list or if it's off by exact amt # so looks like it's intentional, and detach near_miss_points list # so it doesn't get a rewrite in nmp_merged WPT files - # also set the extra field to mark FP/LI items in the .nmp file - extra_field = "" - if nmpline.rstrip() in nmpfplist: - nmpfplist.remove(nmpline.rstrip()) - nmpline += "[MARKED FP]" + logline = nmpline + if li_count: + if li_count == len(w.near_miss_points): + logline += " [LOOKS INTENTIONAL]" + else: + logline += " [SOME LOOK INTENTIONAL]" w.near_miss_points = None - extra_field += "FP" - if nmplooksintentional: - nmpline += "[LOOKS INTENTIONAL]" + if fp: + if lifp_tag == 0: + nmpfps.remove(nmpline) + elif lifp_tag == 1: + nmpfps.remove(nmpline+" [LOOKS INTENTIONAL]") + else: + nmpfps.remove(nmpline+" [SOME LOOK INTENTIONAL]") + logline += " [MARKED FP]" w.near_miss_points = None - extra_field += "LI" - if extra_field != "": - extra_field = " " + extra_field - nmploglines.append(nmpline.rstrip()) - - # write actual lines to .nmp file, indicating FP and/or LI - # for marked FPs or looks intentional items - for nmpnmpline in nmpnmplines: - nmpnmp.write(nmpnmpline + extra_field + "\n") + nmploglines.append(logline) nmpnmp.close() # sort and write actual lines to nearmisspoints.log @@ -2479,9 +2505,10 @@ def run(self): # report any unmatched nmpfps.log entries nmpfpsunmatchedfile = open(args.logfilepath+'/nmpfpsunmatched.log','w') -for line in nmpfplist: +for line in sorted(nmpfps): nmpfpsunmatchedfile.write(line + '\n') nmpfpsunmatchedfile.close() +nmpfps = None # if requested, rewrite data with near-miss points merged in if args.nmpmergepath != "" and not args.errorcheck: