From 73107c87e04f2979a6cc9bb03dfd18704deb3b85 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Wed, 14 Nov 2018 15:46:33 -0500 Subject: [PATCH 01/21] sort datacheck entries --- siteupdate/python-teresco/siteupdate.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index b9028f9a..cea1f296 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -3163,6 +3163,8 @@ def run(self): print("!", flush=True) print(et.et() + "Found " + str(len(datacheckerrors)) + " datacheck errors.") +datacheckerrors.sort(key=lambda DatacheckEntry: str(DatacheckEntry)) + # now mark false positives print(et.et() + "Marking datacheck false positives.",end="",flush=True) fpfile = open(args.logfilepath+'/nearmatchfps.log','w',encoding='utf-8') From 150c742fc7eb46fe02df1e7c42dac51c5e8d7d83 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Wed, 14 Nov 2018 22:26:13 -0500 Subject: [PATCH 02/21] delete waypoint_naming_log when done gain a small amount of RAM headroom near the end of the process --- siteupdate/python-teresco/siteupdate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index cea1f296..118c5f85 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2791,6 +2791,7 @@ def run(self): for line in graph_data.waypoint_naming_log: logfile.write(line + '\n') logfile.close() +graph_data.waypoint_naming_log = None # create list of graph information for the DB graph_list = [] From 41ee5363b4427bfd91fa744ce8f1f340c799e134 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Sat, 24 Nov 2018 01:09:33 -0500 Subject: [PATCH 03/21] concurrency detection fix --- siteupdate/python-teresco/siteupdate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 118c5f85..4b5b5050 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2364,6 +2364,7 @@ def run(self): s.concurrent.append(other) concurrencyfile.write("New concurrency [" + str(s) + "][" + str(other) + "] (" + str(len(s.concurrent)) + ")\n") else: + other.concurrent = s.concurrent if other not in s.concurrent: s.concurrent.append(other) #concurrencyfile.write("Added concurrency [" + str(s) + "]-[" + str(other) + "] ("+ str(len(s.concurrent)) + ")\n") From b8af79317d9ccdb21896b04da34c0a5e12b09658 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Wed, 28 Nov 2018 12:15:32 -0500 Subject: [PATCH 04/21] Sort waypoints in Quadtree for deterministic nearmisspoints.log & tm-master.nmp --- siteupdate/python-teresco/siteupdate.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 4b5b5050..3f85a266 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -233,6 +233,15 @@ def total_nodes(self): else: return 1 + self.nw_child.total_nodes() + self.ne_child.total_nodes() + self.sw_child.total_nodes() + self.se_child.total_nodes() + def sort(self): + if self.points is None: + self.ne_child.sort() + self.nw_child.sort() + self.se_child.sort() + self.sw_child.sort() + else: + self.points.sort(key=lambda waypoint: waypoint.route.root + "@" + waypoint.label) + class Waypoint: """This class encapsulates the information about a single waypoint from a .wpt file. @@ -2146,6 +2155,9 @@ def run(self): #for h in highway_systems: # read_wpts_for_highway_system(h) +print(et.et() + "Sorting waypoints in Quadtree.") +all_waypoints.sort() + print(et.et() + "Finding unprocessed wpt files.", flush=True) unprocessedfile = open(args.logfilepath+'/unprocessedwpts.log','w',encoding='utf-8') if len(all_wpt_files) > 0: From 2038c64bd261e533e2eac5b00acdb1df26655edd Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Wed, 28 Nov 2018 22:41:28 -0500 Subject: [PATCH 05/21] Update to mention localupdate.sh This is now the actual site update process, with the site update running on the same machine as the web server, eliminating the need for file transfers and remote commands over ssh. --- SETUP.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SETUP.md b/SETUP.md index dabeb726..c75c77b4 100644 --- a/SETUP.md +++ b/SETUP.md @@ -26,7 +26,7 @@ python3 siteupdate.py There is also a collection of bash scripts that run this program, typically launched with ``` -sh siteupdate.sh +sh localupdate.sh ``` -that also transfers files to the DB and web servers and reloads the DB. +right on the main server. In addition to running the siteupdate.py program, this also transfers files to the appropriate directories on the web servers and reloads the DB. From cdea81e3e67ae0347e5f7e37b882e31c1d074d35 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Mon, 3 Dec 2018 03:11:41 -0500 Subject: [PATCH 06/21] Sort colocated waypoint lists --- siteupdate/python-teresco/siteupdate.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 3f85a266..1243fdaa 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1155,8 +1155,6 @@ def __init__(self,waypoint_list,datacheckerrors): hid_list.append(w) else: vis_list.append(w) - vis_list.sort(key=lambda waypoint: waypoint.route.root + "@" + waypoint.label) - hid_list.sort(key=lambda waypoint: waypoint.route.root + "@" + waypoint.label) datacheckerrors.append(DatacheckEntry(vis_list[0].route,[vis_list[0].label],"VISIBLE_HIDDEN_COLOC", hid_list[0].route.root+"@"+hid_list[0].label)) @@ -1589,8 +1587,9 @@ def __init__(self, all_waypoints, highway_systems, datacheckerrors): vinfo.is_hidden = False continue if len(vinfo.incident_collapsed_edges) > 2: - dc_waypoint = sorted(vinfo.first_waypoint.colocated, key=lambda waypoint: waypoint.route.root + "@" + waypoint.label)[0] - datacheckerrors.append(DatacheckEntry(dc_waypoint.route,[dc_waypoint.label],"HIDDEN_JUNCTION",str(len(vinfo.incident_collapsed_edges)))) + datacheckerrors.append(DatacheckEntry(vinfo.first_waypoint.colocated[0].route, + [vinfo.first_waypoint.colocated[0].label], + "HIDDEN_JUNCTION",str(len(vinfo.incident_collapsed_edges)))) vinfo.is_hidden = False continue # construct from vertex_info this time @@ -2158,6 +2157,11 @@ def run(self): print(et.et() + "Sorting waypoints in Quadtree.") all_waypoints.sort() +print(et.et() + "Sorting colocated point lists.") +for w in all_waypoints.point_list(): + if w.colocated is not None: + w.colocated.sort(key=lambda waypoint: waypoint.route.root + "@" + waypoint.label) + print(et.et() + "Finding unprocessed wpt files.", flush=True) unprocessedfile = open(args.logfilepath+'/unprocessedwpts.log','w',encoding='utf-8') if len(all_wpt_files) > 0: From 8dda06f619a14adca026938af0112524d3d7e36c Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Thu, 6 Dec 2018 08:04:27 -0500 Subject: [PATCH 07/21] Reorder file installs to after primary DB update so files and DB are (almost) always consistent. --- siteupdate/python-teresco/localupdate.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/siteupdate/python-teresco/localupdate.sh b/siteupdate/python-teresco/localupdate.sh index eee634c9..5a3cf502 100644 --- a/siteupdate/python-teresco/localupdate.sh +++ b/siteupdate/python-teresco/localupdate.sh @@ -82,6 +82,14 @@ if [ "$graphflag" != "-k" ]; then cd - fi +echo "$0: loading primary DB" +date +mysql --defaults-group-suffix=tmapadmin -u travmapadmin TravelMapping < TravelMapping-$datestr.sql +/bin/rm $tmwebbase/dbupdating +echo "$0: switching to primary DB" +date +ln -sf $tmwebbase/lib/tm.conf.standard $tmwebbase/lib/tm.conf + echo "$0: installing logs, stats, nmp_merged, graphs, archiving old contents in $tmpdir/$datestr" mkdir -p $tmpdir/$datestr mv $tmwebbase/$logdir $tmpdir/$datestr @@ -95,13 +103,7 @@ if [ "$graphflag" != "-k" ]; then mv $datestr/$graphdir $tmwebbase fi rmdir $datestr -echo "$0: loading primary DB" -date -mysql --defaults-group-suffix=tmapadmin -u travmapadmin TravelMapping < TravelMapping-$datestr.sql -/bin/rm $tmwebbase/dbupdating -echo "$0: switching to primary DB" -date -ln -sf $tmwebbase/lib/tm.conf.standard $tmwebbase/lib/tm.conf + echo "$0: loading DB copy" mysql --defaults-group-suffix=tmapadmin -u travmapadmin TravelMappingCopy < TravelMapping-$datestr.sql echo "$0: moving sql file to archive" From 850e4a71870413f4ecc932c600904de9613f5e23 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Fri, 7 Dec 2018 04:10:25 -0500 Subject: [PATCH 08/21] qtdebug fix --- siteupdate/python-teresco/siteupdate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 1243fdaa..92cc7013 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -77,8 +77,8 @@ def insert(self,w): #print("QTDEBUG: " + str(self) + " insert " + str(w)) if self.points is not None: if self.waypoint_at_same_point(w) is None: + #print("QTDEBUG: " + str(self) + " at " + str(self.unique_locations) + " unique locations") self.unique_locations += 1 - #print("QTDEBUG: " + str(self) + " has " + str(self.unique_locations) + " unique locations") self.points.append(w) if self.unique_locations > 50: # 50 unique points max per quadtree node self.refine() From 12f00bd9ef3378394dff9de74b8e7de10c345932 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Mon, 10 Dec 2018 00:50:08 -0500 Subject: [PATCH 09/21] nearmatchfps.log formatting --- siteupdate/python-teresco/siteupdate.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 92cc7013..c4e918da 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -3202,9 +3202,8 @@ def run(self): toremove.append(fp) break if d.match_except_info(fp): - fpfile.write("DCERROR: " + str(d) + "\n") - fpfile.write("FPENTRY: " + fp[0] + ';' + fp[1] + ';' + fp[2] + ';' + fp[3] + ';' + fp[4] + ';' + fp[5] + '\n') - fpfile.write("REPLACEWITH: " + fp[0] + ';' + fp[1] + ';' + fp[2] + ';' + fp[3] + ';' + fp[4] + ';' + d.info + '\n') + fpfile.write("FP_ENTRY: " + fp[0] + ';' + fp[1] + ';' + fp[2] + ';' + fp[3] + ';' + fp[4] + ';' + fp[5] + '\n') + fpfile.write("CHANGETO: " + fp[0] + ';' + fp[1] + ';' + fp[2] + ';' + fp[3] + ';' + fp[4] + ';' + d.info + '\n') fpfile.close() # now remove the ones we matched from the list From ed5e912dd5b0f33a5c89e8cf677352246757faed Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Mon, 10 Dec 2018 01:47:12 -0500 Subject: [PATCH 10/21] use .list line formatting in str(HighwaySegment) --- siteupdate/python-teresco/siteupdate.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index c4e918da..28afbcf1 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -601,8 +601,7 @@ def __init__(self,w1,w2,route): self.segment_name = None def __str__(self): - return self.waypoint1.label + " to " + self.waypoint2.label + \ - " via " + self.route.root + return self.route.region + " " + self.route.route + " " + self.waypoint1.label + " " + self.waypoint2.label def add_clinched_by(self,traveler): if traveler not in self.clinched_by: From eccdb9b983447d54686d0179bd1c70224ef9c501 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Mon, 10 Dec 2018 01:55:38 -0500 Subject: [PATCH 11/21] remove unused strings in Route and ConnectedRoute --- siteupdate/python-teresco/siteupdate.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 28afbcf1..8be0f163 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -672,7 +672,6 @@ class Route: def __init__(self,line,system,el): """initialize object from a .csv file line, but do not yet read in waypoint file""" - self.line = line fields = line.split(";") if len(fields) != 8: el.add_error("Could not parse csv line: [" + line + @@ -800,7 +799,6 @@ class ConnectedRoute: def __init__(self,line,system,el): """initialize the object from the _con.csv line given""" - self.line = line fields = line.split(";") if len(fields) != 5: el.add_error("Could not parse _con.csv line: [" + line + From d0cca48a2fd648a795490040eabb275ffe7e7a78 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Mon, 10 Dec 2018 02:11:22 -0500 Subject: [PATCH 12/21] ConnectedRoute.mileage comment --- siteupdate/python-teresco/siteupdate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 8be0f163..85317868 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -830,7 +830,7 @@ def __init__(self,line,system,el): rootOrder += 1 if len(self.roots) < 1: el.add_error("No roots in _con.csv line [" + line + "]") - # will be computed for routes in active systems later + # will be computed for routes in active & preview systems later self.mileage = 0.0 def csv_line(self): From 6b2688c800ba0d8661f4aacc142335cb7a67888e Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Mon, 10 Dec 2018 12:40:53 -0500 Subject: [PATCH 13/21] Duplicate root in con_route: variable names --- siteupdate/python-teresco/siteupdate.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 85317868..9854220e 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2025,12 +2025,12 @@ def __init__(self,filename,descr,vertices,edges,format,category): con_roots = set() for h in highway_systems: - for r in h.con_route_list: - for cr in r.roots: - if cr.root in con_roots: - el.add_error("Duplicate root in con_route lists: " + cr.root) + for cr in h.con_route_list: + for r in cr.roots: + if r.root in con_roots: + el.add_error("Duplicate root in con_route lists: " + r.root) else: - con_roots.add(cr.root) + con_roots.add(r.root) # Make sure every route was listed as a part of some connected route if len(roots) == len(con_roots): From 6c8512ab91dff5436b63ef470e18940047166192 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Mon, 10 Dec 2018 13:03:41 -0500 Subject: [PATCH 14/21] simplify duplicate list name reporting --- siteupdate/python-teresco/siteupdate.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 9854220e..997a4433 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2053,14 +2053,9 @@ def __init__(self,filename,descr,vertices,edges,format,category): # report any duplicate list names as errors if len(duplicate_list_names) > 0: - print("Found " + str(len(duplicate_list_names)) + " DUPLICATE_LIST_NAME cases.") - num_found = 0 - for h in highway_systems: - for r in h.route_list: - if r.region + ' ' + r.list_entry_name() in duplicate_list_names: - el.add_error("Duplicate list name: " + r.region + ' ' + r.list_entry_name()) - num_found += 1 - print("Added " + str(num_found) + " DUPLICATE_LIST_NAME error entries.") + print("Found " + str(len(duplicate_list_names)) + " DUPLICATE_LIST_NAME case(s).") + for d in duplicate_list_names: + el.add_error("Duplicate list name: " + d) else: print("No duplicate list names found.") From 81d87af819c9270e69f0c2d9724631e04575e41f Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Thu, 13 Dec 2018 13:40:31 -0500 Subject: [PATCH 15/21] remove unused arg from TravelerList constructor --- siteupdate/python-teresco/siteupdate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 997a4433..5b051d4b 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -927,7 +927,7 @@ class TravelerList: start_waypoint end_waypoint """ - def __init__(self,travelername,systems,route_hash,path="../../../UserData/list_files"): + def __init__(self,travelername,route_hash,path="../../../UserData/list_files"): self.list_entries = [] self.clinched_segments = set() self.traveler_name = travelername[:-5] @@ -2280,7 +2280,7 @@ def run(self): for t in traveler_ids: if t.endswith('.list'): print(" " + t,end="",flush=True) - traveler_lists.append(TravelerList(t,highway_systems,route_hash,args.userlistfilepath)) + traveler_lists.append(TravelerList(t,route_hash,args.userlistfilepath)) print(" processed " + str(len(traveler_lists)) + " traveler list files.") # Read updates.csv file, just keep in the fields array for now since we're From dc8a05c0968abaee8b136305c5e5118fdcdbb84c Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Fri, 14 Dec 2018 14:27:06 -0500 Subject: [PATCH 16/21] fix str(HighwaySegment) done incorrectly in ed5e912dd5b0f33a5c89e8cf677352246757faed --- siteupdate/python-teresco/siteupdate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 5b051d4b..874e63c3 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -601,7 +601,7 @@ def __init__(self,w1,w2,route): self.segment_name = None def __str__(self): - return self.route.region + " " + self.route.route + " " + self.waypoint1.label + " " + self.waypoint2.label + return self.route.readable_name() + " " + self.waypoint1.label + " " + self.waypoint2.label def add_clinched_by(self,traveler): if traveler not in self.clinched_by: From 36c208235a02fd86e8614e2babe19d6d99f354bd Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Thu, 20 Dec 2018 01:18:47 -0500 Subject: [PATCH 17/21] cope with missing trailing slash --- nmpfilter/nmpbyregion.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/nmpfilter/nmpbyregion.cpp b/nmpfilter/nmpbyregion.cpp index cd3dcaaf..ae1fbbce 100644 --- a/nmpfilter/nmpbyregion.cpp +++ b/nmpfilter/nmpbyregion.cpp @@ -72,6 +72,7 @@ int main(int argc, char *argv[]) // Attempt to find most recent commit info string MasterInfo; string MasterPath = argv[1]; + if (MasterPath.back() != '/') MasterPath += '/'; MasterPath += "../.git/refs/heads/master"; ifstream MasterFile(MasterPath.data()); if (MasterFile) From 9d73c34cd78f33e750b9142333d816e80707dacb Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Fri, 21 Dec 2018 01:58:59 -0500 Subject: [PATCH 18/21] wpt line checks for #57 --- siteupdate/python-teresco/siteupdate.py | 65 +++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 5 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 874e63c3..7a8c4576 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -253,7 +253,7 @@ class Waypoint: root is the unique identifier for the route in which this waypoint is defined """ - def __init__(self,line,route): + def __init__(self,line,route,datacheckerrors): """initialize object from a .wpt file line""" self.route = route parts = line.split() @@ -266,8 +266,61 @@ def __init__(self,line,route): self.alt_labels = [] # last has the URL, which needs more work to get lat/lng url_parts = parts[-1].split('=') + if len(url_parts) < 3: + datacheckerrors.append(DatacheckEntry(route,[self.label],'MALFORMED_URL', parts[-1])) + self.lat = 0 + self.lng = 0 + self.colocated = None + self.near_miss_points = None + return lat_string = url_parts[1].split("&")[0] # chop off "&lon" lng_string = url_parts[2].split("&")[0] # chop off possible "&zoom" + + # make sure lat_string is valid + point_count = 0 + for c in range(len(lat_string)): + # check for multiple decimal points + if lat_string[c] == '.': + point_count += 1 + if point_count > 1: + datacheckerrors.append(DatacheckEntry(route,[self.label],'MALFORMED_URL', parts[-1])) + lat_string = "0" + lng_string = "0" + break + # check for minus sign not at beginning + if lat_string[c] == '-' and c > 0: + datacheckerrors.append(DatacheckEntry(route,[self.label],'MALFORMED_URL', parts[-1])) + lat_string = "0" + lng_string = "0" + break + # check for invalid characters + if lat_string[c] not in "-.0123456789": + datacheckerrors.append(DatacheckEntry(route,[self.label],'MALFORMED_URL', parts[-1])) + lat_string = "0" + lng_string = "0" + break + + # make sure lng_string is valid + point_count = 0 + for c in range(len(lng_string)): + # check for multiple decimal points + if lng_string[c] == '.': + point_count += 1 + if point_count > 1: + datacheckerrors.append(DatacheckEntry(route,[self.label],'MALFORMED_URL', parts[-1])) + lng_string = "0" + break + # check for minus sign not at beginning + if lng_string[c] == '-' and c > 0: + datacheckerrors.append(DatacheckEntry(route,[self.label],'MALFORMED_URL', parts[-1])) + lng_string = "0" + break + # check for invalid characters + if lng_string[c] not in "-.0123456789": + datacheckerrors.append(DatacheckEntry(route,[self.label],'MALFORMED_URL', parts[-1])) + lng_string = "0" + break + self.lat = float(lat_string) self.lng = float(lng_string) # also keep track of a list of colocated waypoints, if any @@ -710,9 +763,10 @@ def read_wpt(self,all_waypoints,all_waypoints_lock,datacheckerrors,el,path="../. file.close() w = None for line in lines: - if len(line.rstrip('\n')) > 0: + line = line.strip() + if len(line) > 0: previous_point = w - w = Waypoint(line.rstrip('\n'),self) + w = Waypoint(line,self,datacheckerrors) self.point_list.append(w) # populate unused alt labels for label in w.alt_labels: @@ -1061,7 +1115,7 @@ class DatacheckEntry: code is the error code string, one of SHARP_ANGLE, BAD_ANGLE, DUPLICATE_LABEL, DUPLICATE_COORDS, LABEL_SELFREF, - LABEL_INVALID_CHAR, LONG_SEGMENT, + LABEL_INVALID_CHAR, LONG_SEGMENT, MALFORMED_URL, LABEL_UNDERSCORES, VISIBLE_DISTANCE, LABEL_PARENS, LACKS_GENERIC, BUS_WITH_I, NONTERMINAL_UNDERSCORE, LONG_UNDERSCORE, LABEL_SLASHES, US_BANNER, VISIBLE_HIDDEN_COLOC, @@ -2986,7 +3040,8 @@ def run(self): datacheckfps = [] datacheck_always_error = [ 'DUPLICATE_LABEL', 'HIDDEN_TERMINUS', 'LABEL_INVALID_CHAR', 'LABEL_SLASHES', - 'LONG_UNDERSCORE', 'NONTERMINAL_UNDERSCORE' ] + 'LONG_UNDERSCORE', 'MALFORMED_URL', + 'NONTERMINAL_UNDERSCORE' ] for line in lines: fields = line.rstrip('\n').split(';') if len(fields) != 6: From b36155db6cacafdf6bd3afffa7cc24b7fb1838cd Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Fri, 28 Dec 2018 02:13:48 -0500 Subject: [PATCH 19/21] bugfix for #153 --- siteupdate/python-teresco/siteupdate.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 7a8c4576..1e62695e 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -308,16 +308,19 @@ def __init__(self,line,route,datacheckerrors): point_count += 1 if point_count > 1: datacheckerrors.append(DatacheckEntry(route,[self.label],'MALFORMED_URL', parts[-1])) + lat_string = "0" lng_string = "0" break # check for minus sign not at beginning if lng_string[c] == '-' and c > 0: datacheckerrors.append(DatacheckEntry(route,[self.label],'MALFORMED_URL', parts[-1])) + lat_string = "0" lng_string = "0" break # check for invalid characters if lng_string[c] not in "-.0123456789": datacheckerrors.append(DatacheckEntry(route,[self.label],'MALFORMED_URL', parts[-1])) + lat_string = "0" lng_string = "0" break From 6622d63c4ed3b58fda46bb5c1fdc78ec622db8cb Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 28 Dec 2018 13:04:49 +0100 Subject: [PATCH 20/21] Highlight the most important output for data check --- siteupdate/python-teresco/siteupdate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 7a8c4576..ab3df305 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -3687,6 +3687,6 @@ def run(self): print("Unique locations: " + str(unique_locations)) if args.errorcheck: - print("Data check successful!") + print("!!! DATA CHECK SUCCESSFUL !!!") print("Total run time: " + et.et()) From 03caed3d16b542c5e3f7718d4954848d5417b42e Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Fri, 28 Dec 2018 22:39:19 -0500 Subject: [PATCH 21/21] Run show processlist from localupdate.sh --- siteupdate/python-teresco/localupdate.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/siteupdate/python-teresco/localupdate.sh b/siteupdate/python-teresco/localupdate.sh index 5a3cf502..be32a089 100644 --- a/siteupdate/python-teresco/localupdate.sh +++ b/siteupdate/python-teresco/localupdate.sh @@ -48,12 +48,13 @@ touch $tmwebbase/dbupdating echo "**********************************************************************" echo "**********************************************************************" echo "* *" -echo "* CHECK FOR USER SLEEP MYSQL PROCESSES USING SHOW PROCESSLIST; *" -echo "* BEFORE THE SITE UPDATE SCRIPT FINISHES TO AVOID A POSSIBLE HANG *" -echo "* DURING INGESTION OF THE NEW .sql FILE. *" +echo "* CHECKING FOR USER SLEEP MYSQL PROCESSES USING SHOW PROCESSLIST; *" +echo "* REMOVE ANY ENTRIES BEFORE THE SITE UPDATE SCRIPT FINISHES TO AVOID *" +echo "* A POSSIBLE HANG DURING INGESTION OF THE NEW .sql FILE. *" echo "* *" echo "**********************************************************************" echo "**********************************************************************" +echo "show processlist;" | mysql --defaults-group-suffix=travmap -u travmap echo "$0: launching siteupdate.py" PYTHONIOENCODING='utf-8' ./siteupdate.py -d TravelMapping-$datestr $graphflag -l $datestr/$logdir -c $datestr/$statdir -g $datestr/$graphdir -n $datestr/$nmpmdir | tee $datestr/$logdir/siteupdate.log 2>&1 || exit 1