From 616ee5dea38accdc8748bc86dd4dd2e42d66f0ec Mon Sep 17 00:00:00 2001 From: eric bryant Date: Sat, 23 Mar 2019 14:14:23 -0400 Subject: [PATCH 01/17] rename classes & variables in preparation for "traveled" graph implementation --- siteupdate/python-teresco/siteupdate.py | 231 ++++++++++++------------ 1 file changed, 116 insertions(+), 115 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 74e906b7..5b65f570 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1190,7 +1190,7 @@ def __str__(self): entry += self.code+";"+self.info return entry -class HighwayGraphVertexInfo: +class HGVertex: """This class encapsulates information needed for a highway graph vertex. """ @@ -1206,8 +1206,8 @@ def __init__(self,wpt,unique_name,datacheckerrors,rg_vset_hash): self.first_waypoint = wpt self.regions = set() self.systems = set() - self.incident_edges = [] - self.incident_collapsed_edges = [] + self.incident_s_edges = [] # simple + self.incident_c_edges = [] # collapsed if wpt.colocated is None: if not wpt.is_hidden: self.is_hidden = False @@ -1276,20 +1276,20 @@ def __init__(self,s,graph): # checks for the very unusual cases where an edge ends up # in the system as itself and its "reverse" duplicate = False - for e in self.vertex1.incident_edges: + for e in self.vertex1.incident_s_edges: if e.vertex1 == self.vertex2 and e.vertex2 == self.vertex1: duplicate = True - for e in self.vertex2.incident_edges: + for e in self.vertex2.incident_s_edges: if e.vertex1 == self.vertex2 and e.vertex2 == self.vertex1: duplicate = True if not duplicate: - self.vertex1.incident_edges.append(self) - self.vertex2.incident_edges.append(self) + self.vertex1.incident_s_edges.append(self) + self.vertex2.incident_s_edges.append(self) else: # flag as invalid/duplicate in order to bypass - # building a HighwayGraphCollapsedEdgeInfo + # building HGEdge self.vertex1 = None # compute an edge label, optionally resticted by systems @@ -1308,14 +1308,14 @@ def label(self,systems=None): def __str__(self): return "HighwayGraphEdgeInfo: " + self.segment_name + " from " + str(self.vertex1) + " to " + str(self.vertex2) -class HighwayGraphCollapsedEdgeInfo: +class HGEdge: """This class encapsulates information needed for a highway graph edge that can incorporate intermediate points. """ - def __init__(self,HGEdge=None,vertex_info=None): - if HGEdge is None and vertex_info is None: - print("ERROR: improper use of HighwayGraphCollapsedEdgeInfo constructor\n") + def __init__(self,ref_edge=None,vertex=None): + if ref_edge is None and vertex is None: + print("ERROR: improper use of HGEdge constructor\n") return # a few items we can do for either construction type @@ -1326,34 +1326,34 @@ def __init__(self,HGEdge=None,vertex_info=None): self.intermediate_points = [] # initial construction is based on a HighwayGraphEdgeInfo - if HGEdge is not None: - self.segment_name = HGEdge.segment_name - self.vertex1 = HGEdge.vertex1 - self.vertex2 = HGEdge.vertex2 + if ref_edge is not None: + self.segment_name = ref_edge.segment_name + self.vertex1 = ref_edge.vertex1 + self.vertex2 = ref_edge.vertex2 # assumption: each edge/segment lives within a unique region # and a 'multi-edge' would not be able to span regions as there # would be a required visible waypoint at the border - self.region = HGEdge.region + self.region = ref_edge.region # a list of route name/system pairs - self.route_names_and_systems = HGEdge.route_names_and_systems - self.vertex1.incident_collapsed_edges.append(self) - self.vertex2.incident_collapsed_edges.append(self) + self.route_names_and_systems = ref_edge.route_names_and_systems + self.vertex1.incident_c_edges.append(self) + self.vertex2.incident_c_edges.append(self) # build by collapsing two existing edges around a common # hidden vertex waypoint, whose information is given in - # vertex_info - if vertex_info is not None: + # vertex + if vertex is not None: # we know there are exactly 2 incident edges, as we # checked for that, and we will replace these two # with the single edge we are constructing here - edge1 = vertex_info.incident_collapsed_edges[0] - edge2 = vertex_info.incident_collapsed_edges[1] + edge1 = vertex.incident_c_edges[0] + edge2 = vertex.incident_c_edges[1] # segment names should match as routes should not start or end # nor should concurrencies begin or end at a hidden point if edge1.segment_name != edge2.segment_name: - print("ERROR: segment name mismatch in HighwayGraphCollapsedEdgeInfo: edge1 named " + edge1.segment_name + " edge2 named " + edge2.segment_name + "\n") + print("ERROR: segment name mismatch in HGEdge collapse constructor: edge1 named " + edge1.segment_name + " edge2 named " + edge2.segment_name + "\n") self.segment_name = edge1.segment_name - #print("\nDEBUG: collapsing edges along " + self.segment_name + " at vertex " + str(vertex_info) + ", edge1 is " + str(edge1) + " and edge2 is " + str(edge2)) + #print("\nDEBUG: collapsing edges along " + self.segment_name + " at vertex " + str(vertex) + ", edge1 is " + str(edge1) + " and edge2 is " + str(edge2)) # region and route names/systems should also match, but not # doing that sanity check here, as the above check should take # care of that @@ -1367,7 +1367,7 @@ def __init__(self,HGEdge=None,vertex_info=None): self.intermediate_points = edge1.intermediate_points.copy() #print("DEBUG: copied edge1 intermediates" + self.intermediate_point_string()) - if edge1.vertex1 == vertex_info: + if edge1.vertex1 == vertex: #print("DEBUG: self.vertex1 getting edge1.vertex2: " + str(edge1.vertex2) + " and reversing edge1 intermediates") self.vertex1 = edge1.vertex2 self.intermediate_points.reverse() @@ -1375,12 +1375,12 @@ def __init__(self,HGEdge=None,vertex_info=None): #print("DEBUG: self.vertex1 getting edge1.vertex1: " + str(edge1.vertex1)) self.vertex1 = edge1.vertex1 - #print("DEBUG: appending to intermediates: " + str(vertex_info)) - self.intermediate_points.append(vertex_info) + #print("DEBUG: appending to intermediates: " + str(vertex)) + self.intermediate_points.append(vertex) toappend = edge2.intermediate_points.copy() #print("DEBUG: copied edge2 intermediates" + edge2.intermediate_point_string()) - if edge2.vertex1 == vertex_info: + if edge2.vertex1 == vertex: #print("DEBUG: self.vertex2 getting edge2.vertex2: " + str(edge2.vertex2)) self.vertex2 = edge2.vertex2 else: @@ -1394,25 +1394,25 @@ def __init__(self,HGEdge=None,vertex_info=None): # replace edge references at our endpoints with ourself removed = 0 - if edge1 in self.vertex1.incident_collapsed_edges: - self.vertex1.incident_collapsed_edges.remove(edge1) + if edge1 in self.vertex1.incident_c_edges: + self.vertex1.incident_c_edges.remove(edge1) removed += 1 - if edge1 in self.vertex2.incident_collapsed_edges: - self.vertex2.incident_collapsed_edges.remove(edge1) + if edge1 in self.vertex2.incident_c_edges: + self.vertex2.incident_c_edges.remove(edge1) removed += 1 if removed != 1: print("ERROR: edge1 " + str(edge1) + " removed from " + str(removed) + " adjacency lists instead of 1.") removed = 0 - if edge2 in self.vertex1.incident_collapsed_edges: - self.vertex1.incident_collapsed_edges.remove(edge2) + if edge2 in self.vertex1.incident_c_edges: + self.vertex1.incident_c_edges.remove(edge2) removed += 1 - if edge2 in self.vertex2.incident_collapsed_edges: - self.vertex2.incident_collapsed_edges.remove(edge2) + if edge2 in self.vertex2.incident_c_edges: + self.vertex2.incident_c_edges.remove(edge2) removed += 1 if removed != 1: print("ERROR: edge2 " + str(edge2) + " removed from " + str(removed) + " adjacency lists instead of 1.") - self.vertex1.incident_collapsed_edges.append(self) - self.vertex2.incident_collapsed_edges.append(self) + self.vertex1.incident_c_edges.append(self) + self.vertex2.incident_c_edges.append(self) # compute an edge label, optionally resticted by systems @@ -1429,18 +1429,18 @@ def label(self,systems=None): # printable string for this edge def __str__(self): - return "HighwayGraphCollapsedEdgeInfo: " + self.segment_name + " from " + str(self.vertex1) + " to " + str(self.vertex2) + " via " + str(len(self.intermediate_points)) + " points" + return "HGEdge: " + self.segment_name + " from " + str(self.vertex1) + " to " + str(self.vertex2) + " via " + str(len(self.intermediate_points)) + " points" # line appropriate for a tmg collapsed edge file def collapsed_tmg_line(self, systems=None): - line = str(self.vertex1.vis_vertex_num) + " " + str(self.vertex2.vis_vertex_num) + " " + self.label(systems) + line = str(self.vertex1.c_vertex_num) + " " + str(self.vertex2.c_vertex_num) + " " + self.label(systems) for intermediate in self.intermediate_points: line += " " + str(intermediate.lat) + " " + str(intermediate.lng) return line # line appropriate for a tmg collapsed edge file, with debug info def debug_tmg_line(self, systems=None): - line = str(self.vertex1.vertex_num) + " [" + self.vertex1.unique_name + "] " + str(self.vertex2.vertex_num) + " [" + self.vertex2.unique_name + "] " + self.label(systems) + line = str(self.vertex1.c_vertex_num) + " [" + self.vertex1.unique_name + "] " + str(self.vertex2.c_vertex_num) + " [" + self.vertex2.unique_name + "] " + self.label(systems) for intermediate in self.intermediate_points: line += " [" + intermediate.unique_name + "] " + str(intermediate.lat) + " " + str(intermediate.lng) return line @@ -1468,13 +1468,13 @@ def __init__(self, place, base, lat, lng, r): self.lng = float(lng) self.r = int(r) - def contains_vertex_info(self, vinfo): - """return whether vinfo's coordinates are within this area""" - # convert to radians to compte distance + def contains_vertex(self, v): + """return whether v's coordinates are within this area""" + # convert to radians to compute distance rlat1 = math.radians(self.lat) rlng1 = math.radians(self.lng) - rlat2 = math.radians(vinfo.lat) - rlng2 = math.radians(vinfo.lng) + rlat2 = math.radians(v.lat) + rlng2 = math.radians(v.lng) ans = math.acos(math.cos(rlat1)*math.cos(rlng1)*math.cos(rlat2)*math.cos(rlng2) +\ math.cos(rlat1)*math.sin(rlng1)*math.cos(rlat2)*math.sin(rlng2) +\ @@ -1483,7 +1483,7 @@ def contains_vertex_info(self, vinfo): def contains_waypoint(self, w): """return whether w is within this area""" - # convert to radians to compte distance + # convert to radians to compute distance rlat1 = math.radians(self.lat) rlng1 = math.radians(self.lng) rlat2 = math.radians(w.lat) @@ -1574,14 +1574,14 @@ def __init__(self, all_waypoints, highway_systems, datacheckerrors, et): # we're good; now add point_name to the set and construct a vertex vertex_names.add(point_name) if w.colocated is None: - self.vertices[w] = HighwayGraphVertexInfo(w, point_name, datacheckerrors, self.rg_vset_hash) + self.vertices[w] = HGVertex(w, point_name, datacheckerrors, self.rg_vset_hash) else: - self.vertices[w] = HighwayGraphVertexInfo(w.colocated[0], point_name, datacheckerrors, self.rg_vset_hash) + self.vertices[w] = HGVertex(w.colocated[0], point_name, datacheckerrors, self.rg_vset_hash) # now that vertices are in place with names, set of unique names is no longer needed vertex_names = None - # add edges, which end up in two separate vertex adjacency lists, + # create edges counter = 0 print("!\n" + et.et() + "Creating edges", end="", flush=True) for h in highway_systems: @@ -1598,63 +1598,64 @@ def __init__(self, all_waypoints, highway_systems, datacheckerrors, et): # and again for a graph where hidden waypoints # are merged into the edge structures if e.vertex1 is not None: - HighwayGraphCollapsedEdgeInfo(HGEdge=e) + HGEdge(ref_edge=e) else: e = None # compress edges adjacent to hidden vertices counter = 0 print("!\n" + et.et() + "Compressing collapsed edges", end="", flush=True) - for label, vinfo in self.vertices.items(): + for label, v in self.vertices.items(): if counter % 10000 == 0: print('.', end="", flush=True) counter += 1 - if vinfo.is_hidden: - if len(vinfo.incident_collapsed_edges) < 2: - # these cases are flagged as HIDDEN_TERMINUS - vinfo.is_hidden = False + if v.is_hidden: + # cases with only one edge are flagged as HIDDEN_TERMINUS + if len(v.incident_c_edges) < 2: + v.is_hidden = False continue - if len(vinfo.incident_collapsed_edges) > 2: - datacheckerrors.append(DatacheckEntry(vinfo.first_waypoint.colocated[0].route, - [vinfo.first_waypoint.colocated[0].label], - "HIDDEN_JUNCTION",str(len(vinfo.incident_collapsed_edges)))) - vinfo.is_hidden = False + # if >2 edges, flag HIDDEN_JUNCTION, mark as visible, and do not compress + if len(v.incident_c_edges) > 2: + datacheckerrors.append(DatacheckEntry(v.first_waypoint.colocated[0].route, + [v.first_waypoint.colocated[0].label], + "HIDDEN_JUNCTION",str(len(v.incident_c_edges)))) + v.is_hidden = False continue - # construct from vertex_info this time - HighwayGraphCollapsedEdgeInfo(vertex_info=vinfo) + # construct from vertex this time + HGEdge(vertex=v) # print summary info print("!\n" + et.et() + " Simple graph has " + str(len(self.vertices)) + - " vertices, " + str(self.edge_count()) + " edges.") - print(et.et() + "Collapsed graph has " + str(self.num_visible_vertices()) + + " vertices, " + str(self.simple_edge_count()) + " edges.") + print(et.et() + "Collapsed graph has " + str(self.num_collapsed_vertices()) + " vertices, " + str(self.collapsed_edge_count()) + " edges.") - def num_visible_vertices(self): + def num_collapsed_vertices(self): count = 0 for v in self.vertices.values(): if not v.is_hidden: count += 1 return count - def edge_count(self): + def simple_edge_count(self): edges = 0 for v in self.vertices.values(): - edges += len(v.incident_edges) + edges += len(v.incident_s_edges) return edges//2 def collapsed_edge_count(self): edges = 0 for v in self.vertices.values(): if not v.is_hidden: - edges += len(v.incident_collapsed_edges) + edges += len(v.incident_c_edges) return edges//2 def matching_vertices(self, regions, systems, placeradius, rg_vset_hash): # return a tuple containing # 1st, a set of vertices from the graph, optionally # restricted by region or system or placeradius area - # 2nd, the number of visible vertices in this set - visible = 0 + # 2nd, the number of collapsed vertices in this set + cv_count = 0 vertex_set = set() rg_vertex_set = set() sys_vertex_set = set() @@ -1683,28 +1684,28 @@ def matching_vertices(self, regions, systems, placeradius, rg_vset_hash): # neither are populated; include all vertices... for v in self.vertices.values(): # ...unless a PlaceRadius is specified - if placeradius is None or placeradius.contains_vertex_info(v): + if placeradius is None or placeradius.contains_vertex(v): vertex_set.add(v) # if placeradius is provided along with non-empty region # or system parameters, erase vertices outside placeradius if placeradius is not None and (systems is not None or regions is not None): pr_vertex_set = set() for v in vertex_set: - if placeradius.contains_vertex_info(v): + if placeradius.contains_vertex(v): pr_vertex_set.add(v) vertex_set = pr_vertex_set - # find number of visible vertices + # find number of collapsed vertices for v in vertex_set: if not v.is_hidden: - visible += 1 - return (vertex_set, visible) + cv_count += 1 + return (vertex_set, cv_count) - def matching_edges(self, mv, regions=None, systems=None, placeradius=None): + def matching_simple_edges(self, mv, regions=None, systems=None, placeradius=None): # return a set of edges from the graph, optionally # restricted by region or system or placeradius area edge_set = set() for v in mv: - for e in v.incident_edges: + for e in v.incident_s_edges: if placeradius is None or placeradius.contains_edge(e): if regions is None or e.region in regions: system_match = systems is None @@ -1725,7 +1726,7 @@ def matching_collapsed_edges(self, mv, regions=None, systems=None, for v in mv: if v.is_hidden: continue - for e in v.incident_collapsed_edges: + for e in v.incident_c_edges: if placeradius is None or placeradius.contains_edge(e): if regions is None or e.region in regions: system_match = systems is None @@ -1750,59 +1751,59 @@ def matching_collapsed_edges(self, mv, regions=None, systems=None, def write_master_tmg_simple(self,filename): tmgfile = open(filename, 'w') tmgfile.write("TMG 1.0 simple\n") - tmgfile.write(str(len(self.vertices)) + ' ' + str(self.edge_count()) + '\n') + tmgfile.write(str(len(self.vertices)) + ' ' + str(self.simple_edge_count()) + '\n') # number waypoint entries as we go to support original .gra # format output - vertex_num = 0 - for vinfo in self.vertices.values(): - tmgfile.write(vinfo.unique_name + ' ' + str(vinfo.lat) + ' ' + str(vinfo.lng) + '\n') - vinfo.vertex_num = vertex_num - vertex_num += 1 + s_vertex_num = 0 + for v in self.vertices.values(): + tmgfile.write(v.unique_name + ' ' + str(v.lat) + ' ' + str(v.lng) + '\n') + v.s_vertex_num = s_vertex_num + s_vertex_num += 1 # sanity check - if len(self.vertices) != vertex_num: - print("ERROR: computed " + str(len(self.vertices)) + " waypoints but wrote " + str(vertex_num)) + if len(self.vertices) != s_vertex_num: + print("ERROR: computed " + str(len(self.vertices)) + " waypoints but wrote " + str(s_vertex_num)) # now edges, only print if not already printed edge = 0 for v in self.vertices.values(): - for e in v.incident_edges: + for e in v.incident_s_edges: if not e.written: e.written = True - tmgfile.write(str(e.vertex1.vertex_num) + ' ' + str(e.vertex2.vertex_num) + ' ' + e.label() + '\n') + tmgfile.write(str(e.vertex1.s_vertex_num) + ' ' + str(e.vertex2.s_vertex_num) + ' ' + e.label() + '\n') edge += 1 # sanity checks for v in self.vertices.values(): - for e in v.incident_edges: + for e in v.incident_s_edges: if not e.written: - print("ERROR: never wrote edge " + str(e.vertex1.vertex_num) + ' ' + str(e.vertex2.vertex_num) + ' ' + e.label() + '\n') - if self.edge_count() != edge: - print("ERROR: computed " + str(self.edge_count()) + " edges but wrote " + str(edge) + "\n") + print("ERROR: never wrote edge " + str(e.vertex1.s_vertex_num) + ' ' + str(e.vertex2.s_vertex_num) + ' ' + e.label() + '\n') + if self.simple_edge_count() != edge: + print("ERROR: computed " + str(self.simple_edge_count()) + " edges but wrote " + str(edge) + "\n") tmgfile.close() - return (len(self.vertices), self.edge_count()) + return (len(self.vertices), self.simple_edge_count()) # write the entire set of data in the tmg collapsed edge format def write_master_tmg_collapsed(self, filename): tmgfile = open(filename, 'w') tmgfile.write("TMG 1.0 collapsed\n") - tmgfile.write(str(self.num_visible_vertices()) + " " + + tmgfile.write(str(self.num_collapsed_vertices()) + " " + str(self.collapsed_edge_count()) + "\n") # write visible vertices - vis_vertex_num = 0 - for vinfo in self.vertices.values(): - if not vinfo.is_hidden: - vinfo.vis_vertex_num = vis_vertex_num - tmgfile.write(vinfo.unique_name + ' ' + str(vinfo.lat) + ' ' + str(vinfo.lng) + '\n') - vis_vertex_num += 1 + c_vertex_num = 0 + for v in self.vertices.values(): + if not v.is_hidden: + v.c_vertex_num = c_vertex_num + tmgfile.write(v.unique_name + ' ' + str(v.lat) + ' ' + str(v.lng) + '\n') + c_vertex_num += 1 # write collapsed edges edge = 0 for v in self.vertices.values(): if not v.is_hidden: - for e in v.incident_collapsed_edges: + for e in v.incident_c_edges: if not e.written: e.written = True tmgfile.write(e.collapsed_tmg_line() + '\n') @@ -1813,7 +1814,7 @@ def write_master_tmg_collapsed(self, filename): print("ERROR: computed " + str(self.collapsed_edge_count()) + " collapsed edges, but wrote " + str(edge) + "\n") tmgfile.close() - return (self.num_visible_vertices(), self.collapsed_edge_count()) + return (self.num_collapsed_vertices(), self.collapsed_edge_count()) # write a subset of the data, # in both simple and collapsed formats, @@ -1821,18 +1822,18 @@ def write_master_tmg_collapsed(self, filename): # by system in the list if given, # or to within a given area if placeradius is given def write_subgraphs_tmg(self, graph_list, path, root, descr, category, regions, systems, placeradius): - visible = 0 + cv_count = 0 simplefile = open(path+root+"-simple.tmg","w",encoding='utf-8') collapfile = open(path+root+".tmg","w",encoding='utf-8') - (mv, visible) = self.matching_vertices(regions, systems, placeradius, self.rg_vset_hash) - mse = self.matching_edges(mv, regions, systems, placeradius) + (mv, cv_count) = self.matching_vertices(regions, systems, placeradius, self.rg_vset_hash) + mse = self.matching_simple_edges(mv, regions, systems, placeradius) mce = self.matching_collapsed_edges(mv, regions, systems, placeradius) print('(' + str(len(mv)) + ',' + str(len(mse)) + ") ", end="", flush=True) - print('(' + str(visible) + ',' + str(len(mce)) + ") ", end="", flush=True) + print('(' + str(cv_count) + ',' + str(len(mce)) + ") ", end="", flush=True) simplefile.write("TMG 1.0 simple\n") collapfile.write("TMG 1.0 collapsed\n") simplefile.write(str(len(mv)) + ' ' + str(len(mse)) + '\n') - collapfile.write(str(visible) + ' ' + str(len(mce)) + '\n') + collapfile.write(str(cv_count) + ' ' + str(len(mce)) + '\n') # write vertices sv = 0 @@ -1840,23 +1841,23 @@ def write_subgraphs_tmg(self, graph_list, path, root, descr, category, regions, for v in mv: # all vertices, for simple graph simplefile.write(v.unique_name + ' ' + str(v.lat) + ' ' + str(v.lng) + '\n') - v.vertex_num = sv + v.s_vertex_num = sv sv += 1 - # visible vertices, for collapsed graph + # visible vertices for collapsed graph if not v.is_hidden: collapfile.write(v.unique_name + ' ' + str(v.lat) + ' ' + str(v.lng) + '\n') - v.vis_vertex_num = cv + v.c_vertex_num = cv cv += 1 # write edges for e in mse: - simplefile.write(str(e.vertex1.vertex_num) + ' ' + str(e.vertex2.vertex_num) + ' ' + e.label(systems) + '\n') + simplefile.write(str(e.vertex1.s_vertex_num) + ' ' + str(e.vertex2.s_vertex_num) + ' ' + e.label(systems) + '\n') for e in mce: collapfile.write(e.collapsed_tmg_line(systems) + '\n') simplefile.close() collapfile.close() graph_list.append(GraphListEntry(root+"-simple.tmg", descr, len(mv), len(mse), "simple", category)) - graph_list.append(GraphListEntry(root + ".tmg", descr, visible, len(mce), "collapsed", category)) + graph_list.append(GraphListEntry(root + ".tmg", descr, cv_count, len(mce), "collapsed", category)) def format_clinched_mi(clinched,total): """return a nicely-formatted string for a given number of miles From 15c7bd610c526a2cc79b4639f9134b9582de8ad7 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Mon, 25 Mar 2019 10:47:19 -0400 Subject: [PATCH 02/17] use one class for simple & collapsed edge objects --- siteupdate/python-teresco/siteupdate.py | 123 ++++++++---------------- 1 file changed, 39 insertions(+), 84 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 5b65f570..c6fa2412 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1250,92 +1250,54 @@ def __init__(self,wpt,unique_name,datacheckerrors,rg_vset_hash): def __str__(self): return self.unique_name -class HighwayGraphEdgeInfo: - """This class encapsulates information needed for a 'standard' - highway graph edge. - """ - - def __init__(self,s,graph): - # temp debug - self.written = False - self.segment_name = s.segment_name() - self.vertex1 = graph.vertices[s.waypoint1.hashpoint()] - self.vertex2 = graph.vertices[s.waypoint2.hashpoint()] - # assumption: each edge/segment lives within a unique region - self.region = s.route.region - # a list of route name/system pairs - self.route_names_and_systems = [] - if s.concurrent is None: - self.route_names_and_systems.append((s.route.list_entry_name(), s.route.system)) - else: - for cs in s.concurrent: - if cs.route.system.devel(): - continue - self.route_names_and_systems.append((cs.route.list_entry_name(), cs.route.system)) - - # checks for the very unusual cases where an edge ends up - # in the system as itself and its "reverse" - duplicate = False - for e in self.vertex1.incident_s_edges: - if e.vertex1 == self.vertex2 and e.vertex2 == self.vertex1: - duplicate = True - - for e in self.vertex2.incident_s_edges: - if e.vertex1 == self.vertex2 and e.vertex2 == self.vertex1: - duplicate = True - - if not duplicate: - self.vertex1.incident_s_edges.append(self) - self.vertex2.incident_s_edges.append(self) - else: - # flag as invalid/duplicate in order to bypass - # building HGEdge - self.vertex1 = None - - # compute an edge label, optionally resticted by systems - def label(self,systems=None): - the_label = "" - for (name, system) in self.route_names_and_systems: - if systems is None or system in systems: - if the_label == "": - the_label = name - else: - the_label += ","+name - - return the_label - - # printable string for this edge - def __str__(self): - return "HighwayGraphEdgeInfo: " + self.segment_name + " from " + str(self.vertex1) + " to " + str(self.vertex2) - class HGEdge: - """This class encapsulates information needed for a highway graph - edge that can incorporate intermediate points. + """This class encapsulates information needed for a highway graph edge. """ - def __init__(self,ref_edge=None,vertex=None): - if ref_edge is None and vertex is None: - print("ERROR: improper use of HGEdge constructor\n") + def __init__(self,s=None,graph=None,vertex=None): + if s is None and vertex is None: + print("ERROR: improper use of HGEdge constructor: s is None; vertex is None\n") return # a few items we can do for either construction type - self.written = False + self.s_written = False # simple + self.c_written = False # collapsed # intermediate points, if more than 1, will go from vertex1 to # vertex2 self.intermediate_points = [] - # initial construction is based on a HighwayGraphEdgeInfo - if ref_edge is not None: - self.segment_name = ref_edge.segment_name - self.vertex1 = ref_edge.vertex1 - self.vertex2 = ref_edge.vertex2 + # initial construction is based on a HighwaySegment + if s is not None: + if graph is None: + print("ERROR: improper use of HGEdge constructor: s is not None; graph is None\n") + return + self.segment_name = s.segment_name() + self.vertex1 = graph.vertices[s.waypoint1.hashpoint()] + self.vertex2 = graph.vertices[s.waypoint2.hashpoint()] # assumption: each edge/segment lives within a unique region # and a 'multi-edge' would not be able to span regions as there # would be a required visible waypoint at the border - self.region = ref_edge.region + self.region = s.route.region # a list of route name/system pairs - self.route_names_and_systems = ref_edge.route_names_and_systems + self.route_names_and_systems = [] + if s.concurrent is None: + self.route_names_and_systems.append((s.route.list_entry_name(), s.route.system)) + else: + for cs in s.concurrent: + if cs.route.system.devel(): + continue + self.route_names_and_systems.append((cs.route.list_entry_name(), cs.route.system)) + # checks for the very unusual cases where an edge ends up + # in the system as itself and its "reverse" + for e in self.vertex1.incident_s_edges: + if e.vertex1 == self.vertex2 and e.vertex2 == self.vertex1: + return + for e in self.vertex2.incident_s_edges: + if e.vertex1 == self.vertex2 and e.vertex2 == self.vertex1: + return + self.vertex1.incident_s_edges.append(self) + self.vertex2.incident_s_edges.append(self) self.vertex1.incident_c_edges.append(self) self.vertex2.incident_c_edges.append(self) @@ -1593,14 +1555,7 @@ def __init__(self, all_waypoints, highway_systems, datacheckerrors, et): for r in h.route_list: for s in r.segment_list: if s.concurrent is None or s == s.concurrent[0]: - # first one copy for the full simple graph - e = HighwayGraphEdgeInfo(s, self) - # and again for a graph where hidden waypoints - # are merged into the edge structures - if e.vertex1 is not None: - HGEdge(ref_edge=e) - else: - e = None + HGEdge(s, self) # compress edges adjacent to hidden vertices counter = 0 @@ -1768,15 +1723,15 @@ def write_master_tmg_simple(self,filename): edge = 0 for v in self.vertices.values(): for e in v.incident_s_edges: - if not e.written: - e.written = True + if not e.s_written: + e.s_written = True tmgfile.write(str(e.vertex1.s_vertex_num) + ' ' + str(e.vertex2.s_vertex_num) + ' ' + e.label() + '\n') edge += 1 # sanity checks for v in self.vertices.values(): for e in v.incident_s_edges: - if not e.written: + if not e.s_written: print("ERROR: never wrote edge " + str(e.vertex1.s_vertex_num) + ' ' + str(e.vertex2.s_vertex_num) + ' ' + e.label() + '\n') if self.simple_edge_count() != edge: print("ERROR: computed " + str(self.simple_edge_count()) + " edges but wrote " + str(edge) + "\n") @@ -1804,8 +1759,8 @@ def write_master_tmg_collapsed(self, filename): for v in self.vertices.values(): if not v.is_hidden: for e in v.incident_c_edges: - if not e.written: - e.written = True + if not e.c_written: + e.c_written = True tmgfile.write(e.collapsed_tmg_line() + '\n') edge += 1 From 84ddc3075673b3b9a326e710819075ba1ada2a4b Mon Sep 17 00:00:00 2001 From: eric bryant Date: Mon, 25 Mar 2019 11:47:27 -0400 Subject: [PATCH 03/17] concurrent travelers sanity check --- siteupdate/python-teresco/siteupdate.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index c6fa2412..fa82490e 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -704,6 +704,22 @@ def segment_name(self): segment_name += cs.route.list_entry_name() return segment_name + def concurrent_travelers_sanity_check(self): + if self.route.system.devel(): + return "" + if self.concurrent is not None: + for conc in self.concurrent: + if len(self.clinched_by) != len(conc.clinched_by): + if conc.route.system.devel(): + return "" + return '[' + str(self) + ']' + " clinched by " + str(len(self.clinched_by)) + " travelers; " \ + + '[' + str(conc) + ']' + " clinched by " + str(len(conc.clinched_by)) + '\n' + else: + for t in self.clinched_by: + if t not in conc.clinched_by: + return t.traveler_name + " has clinched [" + str(self) + "], but not [" + str(conc) + "]\n" + return "" + class Route: """This class encapsulates the contents of one .csv file line that represents a highway within a system and the corresponding @@ -2378,6 +2394,14 @@ def run(self): print("!") concurrencyfile.close() +"""sanetravfile = open(args.logfilepath+'/concurrent_travelers_sanity_check.log','w',encoding='utf-8') +for h in highway_systems: + for r in h.route_list: + for s in r.segment_list: + sanetravfile.write(s.concurrent_travelers_sanity_check()) +sanetravfile.close() +""" + # compute lots of stats, first total mileage by route, system, overall, where # system and overall are stored in dictionaries by region print(et.et() + "Computing stats.",end="",flush=True) From 46698286184fa2dcf24316caf7e27c6d876f7246 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Mon, 25 Mar 2019 12:35:55 -0400 Subject: [PATCH 04/17] HGEdge: store canonical HighwaySegment instead of region --- siteupdate/python-teresco/siteupdate.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index fa82490e..5f5a1233 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1291,10 +1291,11 @@ def __init__(self,s=None,graph=None,vertex=None): self.segment_name = s.segment_name() self.vertex1 = graph.vertices[s.waypoint1.hashpoint()] self.vertex2 = graph.vertices[s.waypoint2.hashpoint()] + # canonical segment, used to reference region and list of travelers # assumption: each edge/segment lives within a unique region # and a 'multi-edge' would not be able to span regions as there # would be a required visible waypoint at the border - self.region = s.route.region + self.segment = s # a list of route name/system pairs self.route_names_and_systems = [] if s.concurrent is None: @@ -1335,7 +1336,7 @@ def __init__(self,s=None,graph=None,vertex=None): # region and route names/systems should also match, but not # doing that sanity check here, as the above check should take # care of that - self.region = edge1.region + self.segment = edge1.segment self.route_names_and_systems = edge1.route_names_and_systems # figure out and remember which endpoints are not the @@ -1678,7 +1679,7 @@ def matching_simple_edges(self, mv, regions=None, systems=None, placeradius=None for v in mv: for e in v.incident_s_edges: if placeradius is None or placeradius.contains_edge(e): - if regions is None or e.region in regions: + if regions is None or e.segment.route.region in regions: system_match = systems is None if not system_match: for (r, s) in e.route_names_and_systems: @@ -1699,7 +1700,7 @@ def matching_collapsed_edges(self, mv, regions=None, systems=None, continue for e in v.incident_c_edges: if placeradius is None or placeradius.contains_edge(e): - if regions is None or e.region in regions: + if regions is None or e.segment.route.region in regions: system_match = systems is None if not system_match: for (r, s) in e.route_names_and_systems: From 71bbd81407d5b00b2cd76d6f2c9fdd42d3162b35 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Mon, 25 Mar 2019 17:59:47 -0400 Subject: [PATCH 05/17] comment: region -> segment --- siteupdate/python-teresco/siteupdate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 5f5a1233..19c1ca58 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1333,7 +1333,7 @@ def __init__(self,s=None,graph=None,vertex=None): print("ERROR: segment name mismatch in HGEdge collapse constructor: edge1 named " + edge1.segment_name + " edge2 named " + edge2.segment_name + "\n") self.segment_name = edge1.segment_name #print("\nDEBUG: collapsing edges along " + self.segment_name + " at vertex " + str(vertex) + ", edge1 is " + str(edge1) + " and edge2 is " + str(edge2)) - # region and route names/systems should also match, but not + # segment and route names/systems should also match, but not # doing that sanity check here, as the above check should take # care of that self.segment = edge1.segment From e0bc96c52110126abf1cbea2ed476b836290e9f1 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Tue, 26 Mar 2019 15:50:19 -0400 Subject: [PATCH 06/17] HGVertex: is_hidden -> visibility --- siteupdate/python-teresco/siteupdate.py | 26 ++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 19c1ca58..c37387d9 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1216,7 +1216,7 @@ def __init__(self,wpt,unique_name,datacheckerrors,rg_vset_hash): self.lng = wpt.lng self.unique_name = unique_name # will consider hidden iff all colocated waypoints are hidden - self.is_hidden = True + self.visibility = 0 # note: if saving the first waypoint, no longer need # lat & lng and can replace with methods self.first_waypoint = wpt @@ -1226,7 +1226,7 @@ def __init__(self,wpt,unique_name,datacheckerrors,rg_vset_hash): self.incident_c_edges = [] # collapsed if wpt.colocated is None: if not wpt.is_hidden: - self.is_hidden = False + self.visibility = 2 self.regions.add(wpt.route.region) self.systems.add(wpt.route.system) wpt.route.system.vertices.add(self) @@ -1238,7 +1238,7 @@ def __init__(self,wpt,unique_name,datacheckerrors,rg_vset_hash): return for w in wpt.colocated: if not w.is_hidden: - self.is_hidden = False + self.visibility = 2 self.regions.add(w.route.region) self.systems.add(w.route.system) if w.route.region not in rg_vset_hash: @@ -1581,17 +1581,17 @@ def __init__(self, all_waypoints, highway_systems, datacheckerrors, et): if counter % 10000 == 0: print('.', end="", flush=True) counter += 1 - if v.is_hidden: + if v.visibility == 0: # cases with only one edge are flagged as HIDDEN_TERMINUS if len(v.incident_c_edges) < 2: - v.is_hidden = False + v.visibility = 2 continue # if >2 edges, flag HIDDEN_JUNCTION, mark as visible, and do not compress if len(v.incident_c_edges) > 2: datacheckerrors.append(DatacheckEntry(v.first_waypoint.colocated[0].route, [v.first_waypoint.colocated[0].label], "HIDDEN_JUNCTION",str(len(v.incident_c_edges)))) - v.is_hidden = False + v.visibility = 2 continue # construct from vertex this time HGEdge(vertex=v) @@ -1605,7 +1605,7 @@ def __init__(self, all_waypoints, highway_systems, datacheckerrors, et): def num_collapsed_vertices(self): count = 0 for v in self.vertices.values(): - if not v.is_hidden: + if v.visibility == 2: count += 1 return count @@ -1618,7 +1618,7 @@ def simple_edge_count(self): def collapsed_edge_count(self): edges = 0 for v in self.vertices.values(): - if not v.is_hidden: + if v.visibility == 2: edges += len(v.incident_c_edges) return edges//2 @@ -1668,7 +1668,7 @@ def matching_vertices(self, regions, systems, placeradius, rg_vset_hash): vertex_set = pr_vertex_set # find number of collapsed vertices for v in vertex_set: - if not v.is_hidden: + if v.visibility == 2: cv_count += 1 return (vertex_set, cv_count) @@ -1696,7 +1696,7 @@ def matching_collapsed_edges(self, mv, regions=None, systems=None, # placeradius area edge_set = set() for v in mv: - if v.is_hidden: + if v.visibility < 2: continue for e in v.incident_c_edges: if placeradius is None or placeradius.contains_edge(e): @@ -1766,7 +1766,7 @@ def write_master_tmg_collapsed(self, filename): # write visible vertices c_vertex_num = 0 for v in self.vertices.values(): - if not v.is_hidden: + if v.visibility == 2: v.c_vertex_num = c_vertex_num tmgfile.write(v.unique_name + ' ' + str(v.lat) + ' ' + str(v.lng) + '\n') c_vertex_num += 1 @@ -1774,7 +1774,7 @@ def write_master_tmg_collapsed(self, filename): # write collapsed edges edge = 0 for v in self.vertices.values(): - if not v.is_hidden: + if v.visibility == 2: for e in v.incident_c_edges: if not e.c_written: e.c_written = True @@ -1816,7 +1816,7 @@ def write_subgraphs_tmg(self, graph_list, path, root, descr, category, regions, v.s_vertex_num = sv sv += 1 # visible vertices for collapsed graph - if not v.is_hidden: + if v.visibility == 2: collapfile.write(v.unique_name + ' ' + str(v.lat) + ' ' + str(v.lng) + '\n') v.c_vertex_num = cv cv += 1 From 45cc586f73efe6f0fd1b6912556a1e31165f7939 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Tue, 26 Mar 2019 21:56:29 -0400 Subject: [PATCH 07/17] tm-master-traveled.tmg partial implementation * Separate collection of "traveled" edges implemented. * fmt_mask allows new collapsed/traveled edges to be constructed together or separately as needed. * HighwayGraph.matching_vertices calculates & returns # of vertices for traveled graph, though this info is not used yet. * Traveled graph partially implemented: for now, just a glorified collapsed graph with a few extra vertices & edges. * Traveler numbers and hex codes are not yet implemented. * Subgraphs are not yet implemented.. --- siteupdate/python-teresco/siteupdate.py | 188 +++++++++++++++++++----- 1 file changed, 154 insertions(+), 34 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index c37387d9..4ca5c8cc 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1215,8 +1215,11 @@ def __init__(self,wpt,unique_name,datacheckerrors,rg_vset_hash): self.lat = wpt.lat self.lng = wpt.lng self.unique_name = unique_name - # will consider hidden iff all colocated waypoints are hidden self.visibility = 0 + # permitted values: + # 0: never visible outside of simple graphs + # 1: visible only in traveled graph; hidden in collapsed graph + # 2: visible in both traveled & collapsed graphs # note: if saving the first waypoint, no longer need # lat & lng and can replace with methods self.first_waypoint = wpt @@ -1224,6 +1227,7 @@ def __init__(self,wpt,unique_name,datacheckerrors,rg_vset_hash): self.systems = set() self.incident_s_edges = [] # simple self.incident_c_edges = [] # collapsed + self.incident_t_edges = [] # traveled if wpt.colocated is None: if not wpt.is_hidden: self.visibility = 2 @@ -1237,6 +1241,7 @@ def __init__(self,wpt,unique_name,datacheckerrors,rg_vset_hash): rg_vset_hash[wpt.route.region].add(self) return for w in wpt.colocated: + # will consider hidden iff all colocated waypoints are hidden if not w.is_hidden: self.visibility = 2 self.regions.add(w.route.region) @@ -1270,7 +1275,7 @@ class HGEdge: """This class encapsulates information needed for a highway graph edge. """ - def __init__(self,s=None,graph=None,vertex=None): + def __init__(self,s=None,graph=None,vertex=None,fmt_mask=None): if s is None and vertex is None: print("ERROR: improper use of HGEdge constructor: s is None; vertex is None\n") return @@ -1278,6 +1283,7 @@ def __init__(self,s=None,graph=None,vertex=None): # a few items we can do for either construction type self.s_written = False # simple self.c_written = False # collapsed + self.t_written = False # traveled # intermediate points, if more than 1, will go from vertex1 to # vertex2 @@ -1317,6 +1323,8 @@ def __init__(self,s=None,graph=None,vertex=None): self.vertex2.incident_s_edges.append(self) self.vertex1.incident_c_edges.append(self) self.vertex2.incident_c_edges.append(self) + self.vertex1.incident_t_edges.append(self) + self.vertex2.incident_t_edges.append(self) # build by collapsing two existing edges around a common # hidden vertex waypoint, whose information is given in @@ -1324,9 +1332,15 @@ def __init__(self,s=None,graph=None,vertex=None): if vertex is not None: # we know there are exactly 2 incident edges, as we # checked for that, and we will replace these two - # with the single edge we are constructing here - edge1 = vertex.incident_c_edges[0] - edge2 = vertex.incident_c_edges[1] + # with the single edge we are constructing here... + if fmt_mask & 1 == 1: + # ...in the compressed graph, and/or... + edge1 = vertex.incident_c_edges[0] + edge2 = vertex.incident_c_edges[1] + if fmt_mask & 2 == 2: + # ...in the traveled graph, as appropriate + edge1 = vertex.incident_t_edges[0] + edge2 = vertex.incident_t_edges[1] # segment names should match as routes should not start or end # nor should concurrencies begin or end at a hidden point if edge1.segment_name != edge2.segment_name: @@ -1372,27 +1386,48 @@ def __init__(self,s=None,graph=None,vertex=None): #print("DEBUG: intermediates complete: from " + str(self.vertex1) + " via " + self.intermediate_point_string() + " to " + str(self.vertex2)) # replace edge references at our endpoints with ourself - removed = 0 - if edge1 in self.vertex1.incident_c_edges: - self.vertex1.incident_c_edges.remove(edge1) - removed += 1 - if edge1 in self.vertex2.incident_c_edges: - self.vertex2.incident_c_edges.remove(edge1) - removed += 1 - if removed != 1: - print("ERROR: edge1 " + str(edge1) + " removed from " + str(removed) + " adjacency lists instead of 1.") - removed = 0 - if edge2 in self.vertex1.incident_c_edges: - self.vertex1.incident_c_edges.remove(edge2) - removed += 1 - if edge2 in self.vertex2.incident_c_edges: - self.vertex2.incident_c_edges.remove(edge2) - removed += 1 - if removed != 1: - print("ERROR: edge2 " + str(edge2) + " removed from " + str(removed) + " adjacency lists instead of 1.") - self.vertex1.incident_c_edges.append(self) - self.vertex2.incident_c_edges.append(self) - + if fmt_mask & 1 == 1: # collapsed edges + removed = 0 + if edge1 in self.vertex1.incident_c_edges: + self.vertex1.incident_c_edges.remove(edge1) + removed += 1 + if edge1 in self.vertex2.incident_c_edges: + self.vertex2.incident_c_edges.remove(edge1) + removed += 1 + if removed != 1: + print("ERROR: collapsed edge1 " + str(edge1) + " removed from " + str(removed) + " adjacency lists instead of 1.") + removed = 0 + if edge2 in self.vertex1.incident_c_edges: + self.vertex1.incident_c_edges.remove(edge2) + removed += 1 + if edge2 in self.vertex2.incident_c_edges: + self.vertex2.incident_c_edges.remove(edge2) + removed += 1 + if removed != 1: + print("ERROR: collapsed edge2 " + str(edge2) + " removed from " + str(removed) + " adjacency lists instead of 1.") + self.vertex1.incident_c_edges.append(self) + self.vertex2.incident_c_edges.append(self) + if fmt_mask & 2 == 2: # traveled edges + removed = 0 + if edge1 in self.vertex1.incident_t_edges: + self.vertex1.incident_t_edges.remove(edge1) + removed += 1 + if edge1 in self.vertex2.incident_t_edges: + self.vertex2.incident_t_edges.remove(edge1) + removed += 1 + if removed != 1: + print("ERROR: traveled edge1 " + str(edge1) + " removed from " + str(removed) + " adjacency lists instead of 1.") + removed = 0 + if edge2 in self.vertex1.incident_t_edges: + self.vertex1.incident_t_edges.remove(edge2) + removed += 1 + if edge2 in self.vertex2.incident_t_edges: + self.vertex2.incident_t_edges.remove(edge2) + removed += 1 + if removed != 1: + print("ERROR: traveled edge2 " + str(edge2) + " removed from " + str(removed) + " adjacency lists instead of 1.") + self.vertex1.incident_t_edges.append(self) + self.vertex2.incident_t_edges.append(self) # compute an edge label, optionally resticted by systems def label(self,systems=None): @@ -1417,6 +1452,14 @@ def collapsed_tmg_line(self, systems=None): line += " " + str(intermediate.lat) + " " + str(intermediate.lng) return line + # line appropriate for a tmg traveled edge file + def traveled_tmg_line(self, systems=None): + line = str(self.vertex1.t_vertex_num) + " " + str(self.vertex2.t_vertex_num) + " " + self.label(systems) + # TODO TODO TODO + for intermediate in self.intermediate_points: + line += " " + str(intermediate.lat) + " " + str(intermediate.lng) + return line + # line appropriate for a tmg collapsed edge file, with debug info def debug_tmg_line(self, systems=None): line = str(self.vertex1.c_vertex_num) + " [" + self.vertex1.unique_name + "] " + str(self.vertex2.c_vertex_num) + " [" + self.vertex2.unique_name + "] " + self.label(systems) @@ -1484,9 +1527,10 @@ class HighwayGraph: On construction, build a set of unique vertex names and determine edges, at most one per concurrent segment. - Create two sets of edges - one for the full graph - and one for the graph with hidden waypoints compressed into - multi-point edges. + Create three sets of edges: + - one for the simple graph + - one for the collapsed graph with hidden waypoints compressed into multi-point edges + - one for the traveled graph: collapsed edges split at endpoints of users' travels """ def __init__(self, all_waypoints, highway_systems, datacheckerrors, et): @@ -1593,8 +1637,28 @@ def __init__(self, all_waypoints, highway_systems, datacheckerrors, et): "HIDDEN_JUNCTION",str(len(v.incident_c_edges)))) v.visibility = 2 continue + # if edge clinched_by lists mismatch, set visibility to 1 + # (visible in traveled graph; hidden in collapsed graph) + # first, the easy check, for whether set sizes mismatch + if len(v.incident_t_edges[0].segment.clinched_by) \ + != len(v.incident_t_edges[1].segment.clinched_by): + v.visibility = 1 + # next, compare clinched_by lists; look for any element in the 1st not in the 2nd + else: + for t in v.incident_t_edges[0].segment.clinched_by: + if t not in v.incident_t_edges[1].segment.clinched_by: + v.visibility = 1 + break # construct from vertex this time - HGEdge(vertex=v) + if v.visibility == 1: + HGEdge(vertex=v, fmt_mask=1) + else: + if (v.incident_c_edges[0] == v.incident_t_edges[0] and v.incident_c_edges[1] == v.incident_t_edges[1]) \ + or (v.incident_c_edges[0] == v.incident_t_edges[1] and v.incident_c_edges[1] == v.incident_t_edges[0]): + HGEdge(vertex=v, fmt_mask=3) + else: + HGEdge(vertex=v, fmt_mask=1) + HGEdge(vertex=v, fmt_mask=2) # print summary info print("!\n" + et.et() + " Simple graph has " + str(len(self.vertices)) + @@ -1609,6 +1673,13 @@ def num_collapsed_vertices(self): count += 1 return count + def num_traveled_vertices(self): + count = 0 + for v in self.vertices.values(): + if v.visibility >= 1: + count += 1 + return count + def simple_edge_count(self): edges = 0 for v in self.vertices.values(): @@ -1622,12 +1693,21 @@ def collapsed_edge_count(self): edges += len(v.incident_c_edges) return edges//2 + def traveled_edge_count(self): + edges = 0 + for v in self.vertices.values(): + if v.visibility >= 1: + edges += len(v.incident_t_edges) + return edges//2 + def matching_vertices(self, regions, systems, placeradius, rg_vset_hash): # return a tuple containing # 1st, a set of vertices from the graph, optionally # restricted by region or system or placeradius area # 2nd, the number of collapsed vertices in this set + # 3rd, the number of traveled vertices in this set cv_count = 0 + tv_count = 0 vertex_set = set() rg_vertex_set = set() sys_vertex_set = set() @@ -1668,9 +1748,11 @@ def matching_vertices(self, regions, systems, placeradius, rg_vset_hash): vertex_set = pr_vertex_set # find number of collapsed vertices for v in vertex_set: - if v.visibility == 2: - cv_count += 1 - return (vertex_set, cv_count) + if v.visibility >= 1: + tv_count += 1 + if v.visibility == 2: + cv_count += 1 + return (vertex_set, cv_count, tv_count) def matching_simple_edges(self, mv, regions=None, systems=None, placeradius=None): # return a set of edges from the graph, optionally @@ -1788,6 +1870,38 @@ def write_master_tmg_collapsed(self, filename): tmgfile.close() return (self.num_collapsed_vertices(), self.collapsed_edge_count()) + # write the entire set of data in the tmg traveled format + def write_master_tmg_traveled(self, filename): + tmgfile = open(filename, 'w') + tmgfile.write("TMG 2.0 traveled\n") + tmgfile.write(str(self.num_traveled_vertices()) + " " + + str(self.traveled_edge_count()) + "\n") + + # write visible vertices + t_vertex_num = 0 + for v in self.vertices.values(): + if v.visibility >= 1: + v.t_vertex_num = t_vertex_num + tmgfile.write(v.unique_name + ' ' + str(v.lat) + ' ' + str(v.lng) + '\n') + t_vertex_num += 1 + + # write traveled edges + edge = 0 + for v in self.vertices.values(): + if v.visibility >= 1: + for e in v.incident_t_edges: + if not e.t_written: + e.t_written = True + tmgfile.write(e.traveled_tmg_line() + '\n') + edge += 1 + + # sanity check on edges written + if self.traveled_edge_count() != edge: + print("ERROR: computed " + str(self.traveled_edge_count()) + " traveled edges, but wrote " + str(edge) + "\n") + + tmgfile.close() + return (self.num_traveled_vertices(), self.traveled_edge_count()) + # write a subset of the data, # in both simple and collapsed formats, # restricted by regions in the list if given, @@ -1797,7 +1911,7 @@ def write_subgraphs_tmg(self, graph_list, path, root, descr, category, regions, cv_count = 0 simplefile = open(path+root+"-simple.tmg","w",encoding='utf-8') collapfile = open(path+root+".tmg","w",encoding='utf-8') - (mv, cv_count) = self.matching_vertices(regions, systems, placeradius, self.rg_vset_hash) + (mv, cv_count, tv_count) = self.matching_vertices(regions, systems, placeradius, self.rg_vset_hash) mse = self.matching_simple_edges(mv, regions, systems, placeradius) mce = self.matching_collapsed_edges(mv, regions, systems, placeradius) print('(' + str(len(mv)) + ',' + str(len(mse)) + ") ", end="", flush=True) @@ -2849,9 +2963,15 @@ def run(self): print(et.et() + "Writing master TM simple graph file, tm-master-simple.tmg", flush=True) (sv, se) = graph_data.write_master_tmg_simple(args.graphfilepath+'/tm-master-simple.tmg') graph_list.append(GraphListEntry('tm-master-simple.tmg', 'All Travel Mapping Data', sv, se, 'simple', 'master')) + print(et.et() + "Writing master TM collapsed graph file, tm-master.tmg.", flush=True) (cv, ce) = graph_data.write_master_tmg_collapsed(args.graphfilepath+'/tm-master.tmg') graph_list.append(GraphListEntry('tm-master.tmg', 'All Travel Mapping Data', cv, ce, 'collapsed', 'master')) + + print(et.et() + "Writing master TM traveled graph file, tm-master-traveled.tmg.", flush=True) + (tv, te) = graph_data.write_master_tmg_traveled(args.graphfilepath+'/tm-master-traveled.tmg') + graph_list.append(GraphListEntry('tm-master-traveled.tmg', 'All Travel Mapping Data', tv, te, 'traveled', 'master')) + graph_types.append(['master', 'All Travel Mapping Data', 'These graphs contain all routes currently plotted in the Travel Mapping project.']) From 55bc552073b16cb1aeb0f4ab76f0b743c246f31d Mon Sep 17 00:00:00 2001 From: eric bryant Date: Wed, 27 Mar 2019 02:26:10 -0400 Subject: [PATCH 08/17] traveled subgraph partial implementation * Subgraphs partially implemented: for now, just glorified collapsed graphs with a few extra vertices & edges. * Traveler numbers and hex codes are not yet implemented. --- siteupdate/python-teresco/siteupdate.py | 56 ++++++++++++++++++++----- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 4ca5c8cc..4dbef286 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1773,9 +1773,8 @@ def matching_simple_edges(self, mv, regions=None, systems=None, placeradius=None def matching_collapsed_edges(self, mv, regions=None, systems=None, placeradius=None): - # return a set of edges from the graph edges for the collapsed - # edge format, optionally restricted by region or system or - # placeradius area + # return a set of edges for the collapsed edge format, + # optionally restricted by region or system or placeradius edge_set = set() for v in mv: if v.visibility < 2: @@ -1792,6 +1791,26 @@ def matching_collapsed_edges(self, mv, regions=None, systems=None, edge_set.add(e) return edge_set + def matching_traveled_edges(self, mv, regions=None, systems=None, + placeradius=None): + # return a set of edges for the traveled graph format, + # optionally restricted by region or system or placeradius + edge_set = set() + for v in mv: + if v.visibility < 1: + continue + for e in v.incident_t_edges: + if placeradius is None or placeradius.contains_edge(e): + if regions is None or e.segment.route.region in regions: + system_match = systems is None + if not system_match: + for (r, s) in e.route_names_and_systems: + if s in systems: + system_match = True + if system_match: + edge_set.add(e) + return edge_set + # write the entire set of highway data a format very similar to # the original .gra format. The first line is a header specifying # the format and version number, the second line specifying the @@ -1908,42 +1927,57 @@ def write_master_tmg_traveled(self, filename): # by system in the list if given, # or to within a given area if placeradius is given def write_subgraphs_tmg(self, graph_list, path, root, descr, category, regions, systems, placeradius): - cv_count = 0 simplefile = open(path+root+"-simple.tmg","w",encoding='utf-8') collapfile = open(path+root+".tmg","w",encoding='utf-8') + travelfile = open(path+root+"-traveled.tmg","w",encoding='utf-8') (mv, cv_count, tv_count) = self.matching_vertices(regions, systems, placeradius, self.rg_vset_hash) mse = self.matching_simple_edges(mv, regions, systems, placeradius) mce = self.matching_collapsed_edges(mv, regions, systems, placeradius) + mte = self.matching_traveled_edges(mv, regions, systems, placeradius) print('(' + str(len(mv)) + ',' + str(len(mse)) + ") ", end="", flush=True) print('(' + str(cv_count) + ',' + str(len(mce)) + ") ", end="", flush=True) + print('(' + str(tv_count) + ',' + str(len(mte)) + ") ", end="", flush=True) simplefile.write("TMG 1.0 simple\n") collapfile.write("TMG 1.0 collapsed\n") + travelfile.write("TMG 2.0 traveled\n") simplefile.write(str(len(mv)) + ' ' + str(len(mse)) + '\n') collapfile.write(str(cv_count) + ' ' + str(len(mce)) + '\n') + travelfile.write(str(tv_count) + ' ' + str(len(mte)) + '\n') # write vertices sv = 0 cv = 0 + tv = 0 for v in mv: # all vertices, for simple graph simplefile.write(v.unique_name + ' ' + str(v.lat) + ' ' + str(v.lng) + '\n') v.s_vertex_num = sv sv += 1 - # visible vertices for collapsed graph - if v.visibility == 2: - collapfile.write(v.unique_name + ' ' + str(v.lat) + ' ' + str(v.lng) + '\n') - v.c_vertex_num = cv - cv += 1 + # visible vertices + if v.visibility >= 1: + # for traveled graph + travelfile.write(v.unique_name + ' ' + str(v.lat) + ' ' + str(v.lng) + '\n') + v.t_vertex_num = tv + tv += 1 + if v.visibility == 2: + # for collapsed graph + collapfile.write(v.unique_name + ' ' + str(v.lat) + ' ' + str(v.lng) + '\n') + v.c_vertex_num = cv + cv += 1 # write edges for e in mse: simplefile.write(str(e.vertex1.s_vertex_num) + ' ' + str(e.vertex2.s_vertex_num) + ' ' + e.label(systems) + '\n') for e in mce: collapfile.write(e.collapsed_tmg_line(systems) + '\n') + for e in mte: + travelfile.write(e.traveled_tmg_line(systems) + '\n') simplefile.close() collapfile.close() + travelfile.close() - graph_list.append(GraphListEntry(root+"-simple.tmg", descr, len(mv), len(mse), "simple", category)) - graph_list.append(GraphListEntry(root + ".tmg", descr, cv_count, len(mce), "collapsed", category)) + graph_list.append(GraphListEntry(root+ "-simple.tmg", descr, len(mv), len(mse), "simple", category)) + graph_list.append(GraphListEntry(root+ ".tmg", descr, cv_count, len(mce), "collapsed", category)) + graph_list.append(GraphListEntry(root+"-traveled.tmg", descr, tv_count, len(mte), "traveled", category)) def format_clinched_mi(clinched,total): """return a nicely-formatted string for a given number of miles From cedc846b31de4c722f069205e12ff7cf664960ef Mon Sep 17 00:00:00 2001 From: eric bryant Date: Fri, 29 Mar 2019 00:40:42 -0400 Subject: [PATCH 09/17] full traveled graph implementation --- siteupdate/python-teresco/siteupdate.py | 72 +++++++++++++++++++------ 1 file changed, 55 insertions(+), 17 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 4dbef286..6e9aee17 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -720,6 +720,21 @@ def concurrent_travelers_sanity_check(self): return t.traveler_name + " has clinched [" + str(self) + "], but not [" + str(conc) + "]\n" return "" + def clinchedby_code(self, traveler_lists): + # Return a hexadecimal string encoding which travelers have clinched this segment, for use in "traveled" graph files + # Each character stores info for traveler #n thru traveler #n+3 + # The 1st character stores traveler 0 thru traveler 3, + # The 2nd character stores traveler 4 thru traveler 7, etc. + # For each character, the low-order bit stores traveler n, and the high bit traveler n+3. + + code = "" + clinch_array = [0]*( math.ceil(len(traveler_lists)/4) ) + for t in self.clinched_by: + clinch_array[t.traveler_num // 4] += 2 ** (t.traveler_num % 4) + for c in clinch_array: + code += "0123456789ABCDEF"[c] + return code + class Route: """This class encapsulates the contents of one .csv file line that represents a highway within a system and the corresponding @@ -1453,9 +1468,9 @@ def collapsed_tmg_line(self, systems=None): return line # line appropriate for a tmg traveled edge file - def traveled_tmg_line(self, systems=None): + def traveled_tmg_line(self, traveler_lists, systems=None): line = str(self.vertex1.t_vertex_num) + " " + str(self.vertex2.t_vertex_num) + " " + self.label(systems) - # TODO TODO TODO + line += " " + self.segment.clinchedby_code(traveler_lists) for intermediate in self.intermediate_points: line += " " + str(intermediate.lat) + " " + str(intermediate.lng) return line @@ -1665,6 +1680,8 @@ def __init__(self, all_waypoints, highway_systems, datacheckerrors, et): " vertices, " + str(self.simple_edge_count()) + " edges.") print(et.et() + "Collapsed graph has " + str(self.num_collapsed_vertices()) + " vertices, " + str(self.collapsed_edge_count()) + " edges.") + print(et.et() + " Traveled graph has " + str(self.num_traveled_vertices()) + + " vertices, " + str(self.traveled_edge_count()) + " edges.") def num_collapsed_vertices(self): count = 0 @@ -1773,7 +1790,7 @@ def matching_simple_edges(self, mv, regions=None, systems=None, placeradius=None def matching_collapsed_edges(self, mv, regions=None, systems=None, placeradius=None): - # return a set of edges for the collapsed edge format, + # return a set of edges for the collapsed edge graph format, # optionally restricted by region or system or placeradius edge_set = set() for v in mv: @@ -1890,7 +1907,7 @@ def write_master_tmg_collapsed(self, filename): return (self.num_collapsed_vertices(), self.collapsed_edge_count()) # write the entire set of data in the tmg traveled format - def write_master_tmg_traveled(self, filename): + def write_master_tmg_traveled(self, filename, traveler_lists): tmgfile = open(filename, 'w') tmgfile.write("TMG 2.0 traveled\n") tmgfile.write(str(self.num_traveled_vertices()) + " " + @@ -1911,9 +1928,13 @@ def write_master_tmg_traveled(self, filename): for e in v.incident_t_edges: if not e.t_written: e.t_written = True - tmgfile.write(e.traveled_tmg_line() + '\n') + tmgfile.write(e.traveled_tmg_line(traveler_lists) + '\n') edge += 1 + # traveler names + for t in traveler_lists: + tmgfile.write(t.traveler_name + ' ') + # sanity check on edges written if self.traveled_edge_count() != edge: print("ERROR: computed " + str(self.traveled_edge_count()) + " traveled edges, but wrote " + str(edge) + "\n") @@ -1922,11 +1943,11 @@ def write_master_tmg_traveled(self, filename): return (self.num_traveled_vertices(), self.traveled_edge_count()) # write a subset of the data, - # in both simple and collapsed formats, + # in simple, collapsed and traveled formats, # restricted by regions in the list if given, - # by system in the list if given, + # by systems in the list if given, # or to within a given area if placeradius is given - def write_subgraphs_tmg(self, graph_list, path, root, descr, category, regions, systems, placeradius): + def write_subgraphs_tmg(self, graph_list, path, root, descr, category, regions, systems, placeradius, traveler_lists): simplefile = open(path+root+"-simple.tmg","w",encoding='utf-8') collapfile = open(path+root+".tmg","w",encoding='utf-8') travelfile = open(path+root+"-traveled.tmg","w",encoding='utf-8') @@ -1970,7 +1991,12 @@ def write_subgraphs_tmg(self, graph_list, path, root, descr, category, regions, for e in mce: collapfile.write(e.collapsed_tmg_line(systems) + '\n') for e in mte: - travelfile.write(e.traveled_tmg_line(systems) + '\n') + travelfile.write(e.traveled_tmg_line(traveler_lists, systems) + '\n') + + # traveler names + for t in traveler_lists: + travelfile.write(t.traveler_name + ' ') + simplefile.close() collapfile.close() travelfile.close() @@ -2428,6 +2454,11 @@ def run(self): traveler_lists.append(TravelerList(t,route_hash,args.userlistfilepath)) print(" processed " + str(len(traveler_lists)) + " traveler list files.") traveler_lists.sort(key=lambda TravelerList: TravelerList.traveler_name) +# assign traveler numbers +travnum = 0 +for t in traveler_lists: + t.traveler_num = travnum + travnum += 1 # Read updates.csv file, just keep in the fields array for now since we're # just going to drop this into the DB later anyway @@ -3003,7 +3034,7 @@ def run(self): graph_list.append(GraphListEntry('tm-master.tmg', 'All Travel Mapping Data', cv, ce, 'collapsed', 'master')) print(et.et() + "Writing master TM traveled graph file, tm-master-traveled.tmg.", flush=True) - (tv, te) = graph_data.write_master_tmg_traveled(args.graphfilepath+'/tm-master-traveled.tmg') + (tv, te) = graph_data.write_master_tmg_traveled(args.graphfilepath+'/tm-master-traveled.tmg', traveler_lists) graph_list.append(GraphListEntry('tm-master-traveled.tmg', 'All Travel Mapping Data', tv, te, 'traveled', 'master')) graph_types.append(['master', 'All Travel Mapping Data', @@ -3026,7 +3057,8 @@ def run(self): for a in area_list: print(a.base + '(' + str(a.r) + ') ', end="", flush=True) graph_data.write_subgraphs_tmg(graph_list, args.graphfilepath + "/", a.base + str(a.r) + "-area", - a.place + " (" + str(a.r) + " mi radius)", "area", None, None, a) + a.place + " (" + str(a.r) + " mi radius)", "area", None, None, a, + traveler_lists) graph_types.append(['area', 'Routes Within a Given Radius of a Place', 'These graphs contain all routes currently plotted within the given distance radius of the given place.']) print("!") @@ -3044,7 +3076,8 @@ def run(self): region_type = r[4] print(region_code + ' ', end="",flush=True) graph_data.write_subgraphs_tmg(graph_list, args.graphfilepath + "/", region_code + "-region", - region_name + " (" + region_type + ")", "region", [ region_code ], None, None) + region_name + " (" + region_type + ")", "region", [ region_code ], None, None, + traveler_lists) graph_types.append(['region', 'Routes Within a Single Region', 'These graphs contain all routes currently plotted within the given region.']) print("!") @@ -3068,7 +3101,8 @@ def run(self): if h is not None: print(h.systemname + ' ', end="",flush=True) graph_data.write_subgraphs_tmg(graph_list, args.graphfilepath + "/", h.systemname+"-system", - h.systemname + " (" + h.fullname + ")", "system", None, [ h ], None) + h.systemname + " (" + h.fullname + ")", "system", None, [ h ], None, + traveler_lists) if h is not None: graph_types.append(['system', 'Routes Within a Single Highway System', 'These graphs contain the routes within a single highway system and are not restricted by region.']) @@ -3093,7 +3127,8 @@ def run(self): if h.systemname in selected_systems: systems.append(h) graph_data.write_subgraphs_tmg(graph_list, args.graphfilepath + "/", fields[1], - fields[0], "multisystem", None, systems, None) + fields[0], "multisystem", None, systems, None, + traveler_lists) graph_types.append(['multisystem', 'Routes Within Multiple Highway Systems', 'These graphs contain the routes within a set of highway systems.']) print("!") @@ -3117,7 +3152,8 @@ def run(self): if r[0] in selected_regions and r[0] in active_preview_mileage_by_region: region_list.append(r[0]) graph_data.write_subgraphs_tmg(graph_list, args.graphfilepath + "/", fields[1], - fields[0], "multiregion", region_list, None, None) + fields[0], "multiregion", region_list, None, None, + traveler_lists) graph_types.append(['multiregion', 'Routes Within Multiple Regions', 'These graphs contain the routes within a set of regions.']) print("!") @@ -3136,7 +3172,8 @@ def run(self): if len(region_list) >= 2: print(c[0] + " ", end="", flush=True) graph_data.write_subgraphs_tmg(graph_list, args.graphfilepath + "/", c[0] + "-country", - c[1] + " All Routes in Country", "country", region_list, None, None) + c[1] + " All Routes in Country", "country", region_list, None, None, + traveler_lists) graph_types.append(['country', 'Routes Within a Single Multi-Region Country', 'These graphs contain the routes within a single country that is composed of multiple regions that contain plotted routes. Countries consisting of a single region are represented by their regional graph.']) print("!") @@ -3153,7 +3190,8 @@ def run(self): if len(region_list) >= 1: print(c[0] + " ", end="", flush=True) graph_data.write_subgraphs_tmg(graph_list, args.graphfilepath + "/", c[0] + "-continent", - c[1] + " All Routes on Continent", "continent", region_list, None, None) + c[1] + " All Routes on Continent", "continent", region_list, None, None, + traveler_lists) graph_types.append(['continent', 'Routes Within a Continent', 'These graphs contain the routes on a continent.']) print("!") From d49c2d4ab9dfbf8f25a607b20e2c4156e150d957 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Fri, 29 Mar 2019 12:42:32 -0400 Subject: [PATCH 10/17] remove redundant function --- siteupdate/python-teresco/siteupdate.py | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 6e9aee17..daea5a2c 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1518,23 +1518,10 @@ def contains_vertex(self, v): math.sin(rlat1)*math.sin(rlat2)) * 3963.1 # EARTH_RADIUS; return ans <= self.r - def contains_waypoint(self, w): - """return whether w is within this area""" - # convert to radians to compute distance - rlat1 = math.radians(self.lat) - rlng1 = math.radians(self.lng) - rlat2 = math.radians(w.lat) - rlng2 = math.radians(w.lng) - - ans = math.acos(math.cos(rlat1)*math.cos(rlng1)*math.cos(rlat2)*math.cos(rlng2) +\ - math.cos(rlat1)*math.sin(rlng1)*math.cos(rlat2)*math.sin(rlng2) +\ - math.sin(rlat1)*math.sin(rlat2)) * 3963.1 # EARTH_RADIUS; - return ans <= self.r - def contains_edge(self, e): """return whether both endpoints of edge e are within this area""" - return (self.contains_waypoint(e.vertex1) and - self.contains_waypoint(e.vertex2)) + return (self.contains_vertex(e.vertex1) and + self.contains_vertex(e.vertex2)) class HighwayGraph: """This class implements the capability to create graph @@ -1654,7 +1641,7 @@ def __init__(self, all_waypoints, highway_systems, datacheckerrors, et): continue # if edge clinched_by lists mismatch, set visibility to 1 # (visible in traveled graph; hidden in collapsed graph) - # first, the easy check, for whether set sizes mismatch + # first, the easy check, for whether list sizes mismatch if len(v.incident_t_edges[0].segment.clinched_by) \ != len(v.incident_t_edges[1].segment.clinched_by): v.visibility = 1 From 5b8b8d748f98c6c9b300a10f77e4b4783afe0538 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Sat, 30 Mar 2019 14:19:04 -0400 Subject: [PATCH 11/17] traveled graphs: record only relevant travelers https://github.com/TravelMapping/DataProcessing/issues/199 --- siteupdate/python-teresco/siteupdate.py | 37 +++++++++++++------------ 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index daea5a2c..0d1c2cce 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1800,6 +1800,7 @@ def matching_traveled_edges(self, mv, regions=None, systems=None, # return a set of edges for the traveled graph format, # optionally restricted by region or system or placeradius edge_set = set() + trav_set = set() for v in mv: if v.visibility < 1: continue @@ -1813,7 +1814,9 @@ def matching_traveled_edges(self, mv, regions=None, systems=None, system_match = True if system_match: edge_set.add(e) - return edge_set + for t in e.segment.clinched_by: + trav_set.add(t) + return (edge_set, sorted(trav_set, key=lambda TravelerList: TravelerList.traveler_name)) # write the entire set of highway data a format very similar to # the original .gra format. The first line is a header specifying @@ -1934,14 +1937,21 @@ def write_master_tmg_traveled(self, filename, traveler_lists): # restricted by regions in the list if given, # by systems in the list if given, # or to within a given area if placeradius is given - def write_subgraphs_tmg(self, graph_list, path, root, descr, category, regions, systems, placeradius, traveler_lists): + def write_subgraphs_tmg(self, graph_list, path, root, descr, category, regions, systems, placeradius): simplefile = open(path+root+"-simple.tmg","w",encoding='utf-8') collapfile = open(path+root+".tmg","w",encoding='utf-8') travelfile = open(path+root+"-traveled.tmg","w",encoding='utf-8') (mv, cv_count, tv_count) = self.matching_vertices(regions, systems, placeradius, self.rg_vset_hash) mse = self.matching_simple_edges(mv, regions, systems, placeradius) mce = self.matching_collapsed_edges(mv, regions, systems, placeradius) - mte = self.matching_traveled_edges(mv, regions, systems, placeradius) + (mte, traveler_lists) = self.matching_traveled_edges(mv, regions, systems, placeradius) + """if len(traveler_lists) == 0: + print("\n\nNo travelers in " + root + "\n", flush=True)#""" + # assign traveler numbers + travnum = 0 + for t in traveler_lists: + t.traveler_num = travnum + travnum += 1 print('(' + str(len(mv)) + ',' + str(len(mse)) + ") ", end="", flush=True) print('(' + str(cv_count) + ',' + str(len(mce)) + ") ", end="", flush=True) print('(' + str(tv_count) + ',' + str(len(mte)) + ") ", end="", flush=True) @@ -3044,8 +3054,7 @@ def run(self): for a in area_list: print(a.base + '(' + str(a.r) + ') ', end="", flush=True) graph_data.write_subgraphs_tmg(graph_list, args.graphfilepath + "/", a.base + str(a.r) + "-area", - a.place + " (" + str(a.r) + " mi radius)", "area", None, None, a, - traveler_lists) + a.place + " (" + str(a.r) + " mi radius)", "area", None, None, a) graph_types.append(['area', 'Routes Within a Given Radius of a Place', 'These graphs contain all routes currently plotted within the given distance radius of the given place.']) print("!") @@ -3063,8 +3072,7 @@ def run(self): region_type = r[4] print(region_code + ' ', end="",flush=True) graph_data.write_subgraphs_tmg(graph_list, args.graphfilepath + "/", region_code + "-region", - region_name + " (" + region_type + ")", "region", [ region_code ], None, None, - traveler_lists) + region_name + " (" + region_type + ")", "region", [ region_code ], None, None) graph_types.append(['region', 'Routes Within a Single Region', 'These graphs contain all routes currently plotted within the given region.']) print("!") @@ -3088,8 +3096,7 @@ def run(self): if h is not None: print(h.systemname + ' ', end="",flush=True) graph_data.write_subgraphs_tmg(graph_list, args.graphfilepath + "/", h.systemname+"-system", - h.systemname + " (" + h.fullname + ")", "system", None, [ h ], None, - traveler_lists) + h.systemname + " (" + h.fullname + ")", "system", None, [ h ], None) if h is not None: graph_types.append(['system', 'Routes Within a Single Highway System', 'These graphs contain the routes within a single highway system and are not restricted by region.']) @@ -3114,8 +3121,7 @@ def run(self): if h.systemname in selected_systems: systems.append(h) graph_data.write_subgraphs_tmg(graph_list, args.graphfilepath + "/", fields[1], - fields[0], "multisystem", None, systems, None, - traveler_lists) + fields[0], "multisystem", None, systems, None) graph_types.append(['multisystem', 'Routes Within Multiple Highway Systems', 'These graphs contain the routes within a set of highway systems.']) print("!") @@ -3139,8 +3145,7 @@ def run(self): if r[0] in selected_regions and r[0] in active_preview_mileage_by_region: region_list.append(r[0]) graph_data.write_subgraphs_tmg(graph_list, args.graphfilepath + "/", fields[1], - fields[0], "multiregion", region_list, None, None, - traveler_lists) + fields[0], "multiregion", region_list, None, None) graph_types.append(['multiregion', 'Routes Within Multiple Regions', 'These graphs contain the routes within a set of regions.']) print("!") @@ -3159,8 +3164,7 @@ def run(self): if len(region_list) >= 2: print(c[0] + " ", end="", flush=True) graph_data.write_subgraphs_tmg(graph_list, args.graphfilepath + "/", c[0] + "-country", - c[1] + " All Routes in Country", "country", region_list, None, None, - traveler_lists) + c[1] + " All Routes in Country", "country", region_list, None, None) graph_types.append(['country', 'Routes Within a Single Multi-Region Country', 'These graphs contain the routes within a single country that is composed of multiple regions that contain plotted routes. Countries consisting of a single region are represented by their regional graph.']) print("!") @@ -3177,8 +3181,7 @@ def run(self): if len(region_list) >= 1: print(c[0] + " ", end="", flush=True) graph_data.write_subgraphs_tmg(graph_list, args.graphfilepath + "/", c[0] + "-continent", - c[1] + " All Routes on Continent", "continent", region_list, None, None, - traveler_lists) + c[1] + " All Routes on Continent", "continent", region_list, None, None) graph_types.append(['continent', 'Routes Within a Continent', 'These graphs contain the routes on a continent.']) print("!") From 5635614284c35eeb5eb3cbfcca8ec9ec112e9e91 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Mon, 22 Apr 2019 15:17:05 -0400 Subject: [PATCH 12/17] canonical_waypoint_name: if -> elif In theory, saves a tiny bit of time. In practice, not a noticeable amount. :) --- siteupdate/python-teresco/siteupdate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index daea5a2c..7e323ddc 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -541,7 +541,7 @@ def canonical_waypoint_name(self,log): suffix = colocated[check_index].label[colocated[check_index].label.find('_'):] if colocated[other_index].route.list_entry_name() + suffix == colocated[check_index].label: suffixes[other_index] = suffix - if colocated[check_index].label.startswith(colocated[other_index].route.name_no_abbrev()): + elif colocated[check_index].label.startswith(colocated[other_index].route.name_no_abbrev()): this_match = True if '_' in colocated[check_index].label: suffix = colocated[check_index].label[colocated[check_index].label.find('_'):] From b2de340757a2b1808e255640f733a1326330f20c Mon Sep 17 00:00:00 2001 From: eric bryant Date: Mon, 22 Apr 2019 19:53:25 -0400 Subject: [PATCH 13/17] minor datacheck speed improvements --- siteupdate/python-teresco/siteupdate.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 7e323ddc..94c679b2 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -3208,9 +3208,12 @@ def run(self): for w in r.point_list: # duplicate labels - label_list = w.alt_labels.copy() - label_list.append(w.label) - for label in label_list: + lower_label = w.label.lower().strip("+*") + if lower_label in all_route_labels: + datacheckerrors.append(DatacheckEntry(r,[lower_label],"DUPLICATE_LABEL")) + else: + all_route_labels.add(lower_label) + for label in w.alt_labels: lower_label = label.lower().strip("+*") if lower_label in all_route_labels: datacheckerrors.append(DatacheckEntry(r,[lower_label],"DUPLICATE_LABEL")) @@ -3228,7 +3231,7 @@ def run(self): for other_w in r.point_list: if w == other_w: break - if w.lat == other_w.lat and w.lng == other_w.lng and w.label != other_w.label: + if w.lat == other_w.lat and w.lng == other_w.lng: labels = [] labels.append(other_w.label) labels.append(w.label) From 95969e1c913938771cd69f19f655b60d55bf25af Mon Sep 17 00:00:00 2001 From: eric bryant Date: Mon, 22 Apr 2019 20:04:14 -0400 Subject: [PATCH 14/17] comments for duplicate label datacheck --- siteupdate/python-teresco/siteupdate.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 94c679b2..7c7ce56c 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -3208,11 +3208,13 @@ def run(self): for w in r.point_list: # duplicate labels + # first, check primary label lower_label = w.label.lower().strip("+*") if lower_label in all_route_labels: datacheckerrors.append(DatacheckEntry(r,[lower_label],"DUPLICATE_LABEL")) else: all_route_labels.add(lower_label) + # then check alt labels for label in w.alt_labels: lower_label = label.lower().strip("+*") if lower_label in all_route_labels: From 48ed3df2ab3e5276997a819bab2a4019e9034175 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Tue, 23 Apr 2019 16:02:43 -0400 Subject: [PATCH 15/17] terminal LF in traveled graphs (Python) --- siteupdate/python-teresco/siteupdate.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 7c7ce56c..43cc9ece 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1921,6 +1921,7 @@ def write_master_tmg_traveled(self, filename, traveler_lists): # traveler names for t in traveler_lists: tmgfile.write(t.traveler_name + ' ') + tmgfile.write('\n') # sanity check on edges written if self.traveled_edge_count() != edge: @@ -1983,6 +1984,7 @@ def write_subgraphs_tmg(self, graph_list, path, root, descr, category, regions, # traveler names for t in traveler_lists: travelfile.write(t.traveler_name + ' ') + travelfile.write('\n') simplefile.close() collapfile.close() From e815c31e2ae767f9db71cb6b304cc2141a445a52 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Tue, 23 Apr 2019 17:18:22 -0400 Subject: [PATCH 16/17] finalizing traveled graphs https://github.com/TravelMapping/DataProcessing/issues/199 https://github.com/TravelMapping/DataProcessing/pull/201 --- siteupdate/python-teresco/siteupdate.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index a6fd6fb0..66c72001 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -727,6 +727,8 @@ def clinchedby_code(self, traveler_lists): # The 2nd character stores traveler 4 thru traveler 7, etc. # For each character, the low-order bit stores traveler n, and the high bit traveler n+3. + if len(traveler_lists) == 0: + return "0" code = "" clinch_array = [0]*( math.ceil(len(traveler_lists)/4) ) for t in self.clinched_by: @@ -1901,7 +1903,8 @@ def write_master_tmg_traveled(self, filename, traveler_lists): tmgfile = open(filename, 'w') tmgfile.write("TMG 2.0 traveled\n") tmgfile.write(str(self.num_traveled_vertices()) + " " + - str(self.traveled_edge_count()) + "\n") + str(self.traveled_edge_count()) + " " + + str(len(traveler_lists)) + "\n") # write visible vertices t_vertex_num = 0 @@ -1940,7 +1943,7 @@ def write_master_tmg_traveled(self, filename, traveler_lists): # or to within a given area if placeradius is given def write_subgraphs_tmg(self, graph_list, path, root, descr, category, regions, systems, placeradius): simplefile = open(path+root+"-simple.tmg","w",encoding='utf-8') - collapfile = open(path+root+".tmg","w",encoding='utf-8') + collapfile = open(path+root+"-collapsed.tmg","w",encoding='utf-8') travelfile = open(path+root+"-traveled.tmg","w",encoding='utf-8') (mv, cv_count, tv_count) = self.matching_vertices(regions, systems, placeradius, self.rg_vset_hash) mse = self.matching_simple_edges(mv, regions, systems, placeradius) @@ -1961,7 +1964,7 @@ def write_subgraphs_tmg(self, graph_list, path, root, descr, category, regions, travelfile.write("TMG 2.0 traveled\n") simplefile.write(str(len(mv)) + ' ' + str(len(mse)) + '\n') collapfile.write(str(cv_count) + ' ' + str(len(mce)) + '\n') - travelfile.write(str(tv_count) + ' ' + str(len(mte)) + '\n') + travelfile.write(str(tv_count) + ' ' + str(len(mte)) + ' ' + str(len(traveler_lists)) + '\n') # write vertices sv = 0 @@ -2000,9 +2003,9 @@ def write_subgraphs_tmg(self, graph_list, path, root, descr, category, regions, collapfile.close() travelfile.close() - graph_list.append(GraphListEntry(root+ "-simple.tmg", descr, len(mv), len(mse), "simple", category)) - graph_list.append(GraphListEntry(root+ ".tmg", descr, cv_count, len(mce), "collapsed", category)) - graph_list.append(GraphListEntry(root+"-traveled.tmg", descr, tv_count, len(mte), "traveled", category)) + graph_list.append(GraphListEntry(root+ "-simple.tmg", descr, len(mv), len(mse), "simple", category)) + graph_list.append(GraphListEntry(root+"-collapsed.tmg", descr, cv_count, len(mce), "collapsed", category)) + graph_list.append(GraphListEntry(root+ "-traveled.tmg", descr, tv_count, len(mte), "traveled", category)) def format_clinched_mi(clinched,total): """return a nicely-formatted string for a given number of miles @@ -3028,9 +3031,9 @@ def run(self): (sv, se) = graph_data.write_master_tmg_simple(args.graphfilepath+'/tm-master-simple.tmg') graph_list.append(GraphListEntry('tm-master-simple.tmg', 'All Travel Mapping Data', sv, se, 'simple', 'master')) - print(et.et() + "Writing master TM collapsed graph file, tm-master.tmg.", flush=True) - (cv, ce) = graph_data.write_master_tmg_collapsed(args.graphfilepath+'/tm-master.tmg') - graph_list.append(GraphListEntry('tm-master.tmg', 'All Travel Mapping Data', cv, ce, 'collapsed', 'master')) + print(et.et() + "Writing master TM collapsed graph file, tm-master-collapsed.tmg.", flush=True) + (cv, ce) = graph_data.write_master_tmg_collapsed(args.graphfilepath+'/tm-master-collapsed.tmg') + graph_list.append(GraphListEntry('tm-master-collapsed.tmg', 'All Travel Mapping Data', cv, ce, 'collapsed', 'master')) print(et.et() + "Writing master TM traveled graph file, tm-master-traveled.tmg.", flush=True) (tv, te) = graph_data.write_master_tmg_traveled(args.graphfilepath+'/tm-master-traveled.tmg', traveler_lists) From 0ede2eb9fc883ce742c16e3d422f1072d7c186cc Mon Sep 17 00:00:00 2001 From: eric bryant Date: Thu, 2 May 2019 02:20:09 -0400 Subject: [PATCH 17/17] TravelerList.clinched_by: list -> set * 12% speedup Augmenting travelers for detected concurrent segments * 34% speedup Creating per-traveler stats log entries and augmenting data * 1.7% speedup for subgraph generation. Maybe. --- siteupdate/python-teresco/siteupdate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 66c72001..926f3e63 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -669,14 +669,14 @@ def __init__(self,w1,w2,route): self.waypoint2 = w2 self.route = route self.concurrent = None - self.clinched_by = [] + self.clinched_by = set() def __str__(self): return self.route.readable_name() + " " + self.waypoint1.label + " " + self.waypoint2.label def add_clinched_by(self,traveler): if traveler not in self.clinched_by: - self.clinched_by.append(traveler) + self.clinched_by.add(traveler) return True else: return False @@ -1641,7 +1641,7 @@ def __init__(self, all_waypoints, highway_systems, datacheckerrors, et): "HIDDEN_JUNCTION",str(len(v.incident_c_edges)))) v.visibility = 2 continue - # if edge clinched_by lists mismatch, set visibility to 1 + # if edge clinched_by sets mismatch, set visibility to 1 # (visible in traveled graph; hidden in collapsed graph) # first, the easy check, for whether list sizes mismatch if len(v.incident_t_edges[0].segment.clinched_by) \