From 3f938cbc0339c9ddf9bd101efd1c3dfa3a36ce9a Mon Sep 17 00:00:00 2001 From: eric bryant Date: Sun, 14 Feb 2021 19:09:16 -0500 Subject: [PATCH 1/4] Python edge-compression cleanup yakra#107 --- siteupdate/python-teresco/siteupdate.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index f308abc7..8897a821 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2135,7 +2135,7 @@ def __init__(self, all_waypoints, highway_systems, datacheckerrors, et): # compress edges adjacent to hidden vertices counter = 0 print("!\n" + et.et() + "Compressing collapsed edges", end="", flush=True) - for label, v in self.vertices.items(): + for w, v in self.vertices.items(): if counter % 10000 == 0: print('.', end="", flush=True) counter += 1 @@ -2146,8 +2146,7 @@ def __init__(self, all_waypoints, highway_systems, datacheckerrors, et): continue # if >2 edges, flag HIDDEN_JUNCTION, mark as visible, and do not compress if len(v.incident_c_edges) > 2: - datacheckerrors.append(DatacheckEntry(v.first_waypoint.colocated[0].route, - [v.first_waypoint.colocated[0].label], + datacheckerrors.append(DatacheckEntry(w.colocated[0].route,[w.colocated[0].label], "HIDDEN_JUNCTION",str(len(v.incident_c_edges)))) v.visibility = 2 continue From ac65bc389b28c48eaef53554ec494872d2165888 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Mon, 15 Feb 2021 16:33:23 -0500 Subject: [PATCH 2/4] store & retrieve TMG vertex lines --- siteupdate/python-teresco/siteupdate.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 8897a821..6bea1638 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2307,19 +2307,20 @@ def write_master_graphs_tmg(self, graph_list, path, traveler_lists): cv = 0 tv = 0 for v in self.vertices.values(): + vstr = v.unique_name+' '+str(v.lat)+' '+str(v.lng)+'\n' # all vertices for simple graph - simplefile.write(v.unique_name+' '+str(v.lat)+' '+str(v.lng)+'\n') + simplefile.write(vstr) v.s_vertex_num = sv sv += 1 # visible vertices... if v.visibility >= 1: # for traveled graph, - travelfile.write(v.unique_name+' '+str(v.lat)+' '+str(v.lng)+'\n') + travelfile.write(vstr) v.t_vertex_num = tv tv += 1 if v.visibility == 2: # and for collapsed graph - collapfile.write(v.unique_name+' '+str(v.lat)+' '+str(v.lng)+'\n') + collapfile.write(vstr) v.c_vertex_num = cv cv += 1 # now edges, only write if not already written @@ -2391,19 +2392,20 @@ def write_subgraphs_tmg(self, graph_list, path, root, descr, category, regions, cv = 0 tv = 0 for v in mv: + vstr = v.unique_name + ' ' + str(v.lat) + ' ' + str(v.lng) + '\n' # all vertices, for simple graph - simplefile.write(v.unique_name + ' ' + str(v.lat) + ' ' + str(v.lng) + '\n') + simplefile.write(vstr) v.s_vertex_num = sv sv += 1 # visible vertices if v.visibility >= 1: # for traveled graph - travelfile.write(v.unique_name + ' ' + str(v.lat) + ' ' + str(v.lng) + '\n') + travelfile.write(vstr) v.t_vertex_num = tv tv += 1 if v.visibility == 2: # for collapsed graph - collapfile.write(v.unique_name + ' ' + str(v.lat) + ' ' + str(v.lng) + '\n') + collapfile.write(vstr) v.c_vertex_num = cv cv += 1 # write edges From 315f834302e66636c5002678cba8262cb53ec2c0 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Mon, 15 Feb 2021 17:38:35 -0500 Subject: [PATCH 3/4] nix HGVertex::first_waypoint --- siteupdate/cplusplus/classes/GraphGeneration/HGVertex.cpp | 3 --- siteupdate/cplusplus/classes/GraphGeneration/HGVertex.h | 1 - siteupdate/python-teresco/siteupdate.py | 3 --- 3 files changed, 7 deletions(-) diff --git a/siteupdate/cplusplus/classes/GraphGeneration/HGVertex.cpp b/siteupdate/cplusplus/classes/GraphGeneration/HGVertex.cpp index 797f06c5..540fbc4f 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/HGVertex.cpp +++ b/siteupdate/cplusplus/classes/GraphGeneration/HGVertex.cpp @@ -19,9 +19,6 @@ HGVertex::HGVertex(Waypoint *wpt, const std::string *n, unsigned int numthreads) // 0: never visible outside of simple graphs // 1: visible only in traveled graph; hidden in collapsed graph // 2: visible in both traveled & collapsed graphs - // note: if saving the first waypoint, no longer need - // lat & lng and can replace with methods - first_waypoint = wpt; if (!wpt->colocated) { if (!wpt->is_hidden) visibility = 2; wpt->route->region->vertices.insert(this); diff --git a/siteupdate/cplusplus/classes/GraphGeneration/HGVertex.h b/siteupdate/cplusplus/classes/GraphGeneration/HGVertex.h index 155443d1..9afd8ba8 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/HGVertex.h +++ b/siteupdate/cplusplus/classes/GraphGeneration/HGVertex.h @@ -14,7 +14,6 @@ class HGVertex double lat, lng; const std::string *unique_name; char visibility; - Waypoint *first_waypoint; std::list incident_s_edges; // simple std::list incident_c_edges; // collapsed std::list incident_t_edges; // traveled diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 6bea1638..0f02a969 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1689,9 +1689,6 @@ def __init__(self,wpt,unique_name,datacheckerrors,rg_vset_hash): # 0: never visible outside of simple graphs # 1: visible only in traveled graph; hidden in collapsed graph # 2: visible in both traveled & collapsed graphs - # note: if saving the first waypoint, no longer need - # lat & lng and can replace with methods - self.first_waypoint = wpt self.incident_s_edges = [] # simple self.incident_c_edges = [] # collapsed self.incident_t_edges = [] # traveled From b155944c33c1b5b4470bf1b1b16a42a54fff79ea Mon Sep 17 00:00:00 2001 From: eric bryant Date: Mon, 15 Feb 2021 20:57:18 -0500 Subject: [PATCH 4/4] LABEL_INVALID_CHAR fixes flag once per label; note UTF-8 BOM cases interactive rebase 9197da3f490b07e03352bad955718c475dd82739 --- siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp | 10 ++++++++-- siteupdate/cplusplus/siteupdate.cpp | 4 ++-- siteupdate/python-teresco/siteupdate.py | 9 ++++++--- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp index 9cfe7bdc..818791a1 100644 --- a/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp +++ b/siteupdate/cplusplus/classes/Waypoint/Waypoint.cpp @@ -346,7 +346,11 @@ void Waypoint::label_invalid_char() if ((*c == 42 || *c == 43) && c > label.data() || (*c < 40) || (*c == 44) || (*c > 57 && *c < 65) || (*c == 96) || (*c > 122) || (*c > 90 && *c < 95)) - Datacheck::add(route, label, "", "", "LABEL_INVALID_CHAR", ""); + { if (!strncmp(label.data(), "\xEF\xBB\xBF", 3)) + Datacheck::add(route, label, "", "", "LABEL_INVALID_CHAR", "UTF-8 BOM"); + else Datacheck::add(route, label, "", "", "LABEL_INVALID_CHAR", ""); + break; + } for (std::string& lbl : alt_labels) if (lbl == "*") Datacheck::add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); @@ -354,7 +358,9 @@ void Waypoint::label_invalid_char() if (*c == '+' && c > lbl.data() || *c == '*' && (c > lbl.data()+1 || lbl[0] != '+') || (*c < 40) || (*c == 44) || (*c > 57 && *c < 65) || (*c == 96) || (*c > 122) || (*c > 90 && *c < 95)) - Datacheck::add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); + { Datacheck::add(route, lbl, "", "", "LABEL_INVALID_CHAR", ""); + break; + } } bool Waypoint::label_too_long() diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index 8a906dbd..e61f9b8d 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -1,10 +1,10 @@ // Tab Width = 8 -// Travel Mapping Project, Jim Teresco and Eric Bryant, 2015-2020 +// Travel Mapping Project, Jim Teresco and Eric Bryant, 2015-2021 /* Code to read .csv and .wpt files and prepare for adding to the Travel Mapping Project database. -(c) 2015-2020, Jim Teresco and Eric Bryant +(c) 2015-2021, Jim Teresco and Eric Bryant Original Python version by Jim Teresco, with contributions from Eric Bryant and the TravelMapping team C++ translation by Eric Bryant diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 0f02a969..43dd32aa 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -# Travel Mapping Project, Jim Teresco, 2015-2020 +# Travel Mapping Project, Jim Teresco, 2015-2021 """Python code to read .csv and .wpt files and prepare for adding to the Travel Mapping Project database. -(c) 2015-2020, Jim Teresco, Eric Bryant, and Travel Mapping Project contributors +(c) 2015-2021, Jim Teresco, Eric Bryant, and Travel Mapping Project contributors This module defines classes to represent the contents of a .csv file that lists the highways within a system, and a @@ -3916,7 +3916,10 @@ def run(self): # look for labels with invalid characters if not re.fullmatch('\+?\*?[a-zA-Z0-9()/_\-\.]+', w.label): - datacheckerrors.append(DatacheckEntry(r,[w.label],'LABEL_INVALID_CHAR')) + if w.label.encode('utf-8')[0:3] == b'\xef\xbb\xbf': + datacheckerrors.append(DatacheckEntry(r,[w.label],'LABEL_INVALID_CHAR', "UTF-8 BOM")) + else: + datacheckerrors.append(DatacheckEntry(r,[w.label],'LABEL_INVALID_CHAR')) for a in w.alt_labels: if not re.fullmatch('\+?\*?[a-zA-Z0-9()/_\-\.]+', a): datacheckerrors.append(DatacheckEntry(r,[a],'LABEL_INVALID_CHAR'))