diff --git a/SETUP.md b/SETUP.md index dabeb726..c75c77b4 100644 --- a/SETUP.md +++ b/SETUP.md @@ -26,7 +26,7 @@ python3 siteupdate.py There is also a collection of bash scripts that run this program, typically launched with ``` -sh siteupdate.sh +sh localupdate.sh ``` -that also transfers files to the DB and web servers and reloads the DB. +right on the main server. In addition to running the siteupdate.py program, this also transfers files to the appropriate directories on the web servers and reloads the DB. diff --git a/nmpfilter/nmpbyregion.cpp b/nmpfilter/nmpbyregion.cpp index cd3dcaaf..ae1fbbce 100644 --- a/nmpfilter/nmpbyregion.cpp +++ b/nmpfilter/nmpbyregion.cpp @@ -72,6 +72,7 @@ int main(int argc, char *argv[]) // Attempt to find most recent commit info string MasterInfo; string MasterPath = argv[1]; + if (MasterPath.back() != '/') MasterPath += '/'; MasterPath += "../.git/refs/heads/master"; ifstream MasterFile(MasterPath.data()); if (MasterFile) diff --git a/siteupdate/python-teresco/localupdate.sh b/siteupdate/python-teresco/localupdate.sh index eee634c9..be32a089 100644 --- a/siteupdate/python-teresco/localupdate.sh +++ b/siteupdate/python-teresco/localupdate.sh @@ -48,12 +48,13 @@ touch $tmwebbase/dbupdating echo "**********************************************************************" echo "**********************************************************************" echo "* *" -echo "* CHECK FOR USER SLEEP MYSQL PROCESSES USING SHOW PROCESSLIST; *" -echo "* BEFORE THE SITE UPDATE SCRIPT FINISHES TO AVOID A POSSIBLE HANG *" -echo "* DURING INGESTION OF THE NEW .sql FILE. *" +echo "* CHECKING FOR USER SLEEP MYSQL PROCESSES USING SHOW PROCESSLIST; *" +echo "* REMOVE ANY ENTRIES BEFORE THE SITE UPDATE SCRIPT FINISHES TO AVOID *" +echo "* A POSSIBLE HANG DURING INGESTION OF THE NEW .sql FILE. *" echo "* *" echo "**********************************************************************" echo "**********************************************************************" +echo "show processlist;" | mysql --defaults-group-suffix=travmap -u travmap echo "$0: launching siteupdate.py" PYTHONIOENCODING='utf-8' ./siteupdate.py -d TravelMapping-$datestr $graphflag -l $datestr/$logdir -c $datestr/$statdir -g $datestr/$graphdir -n $datestr/$nmpmdir | tee $datestr/$logdir/siteupdate.log 2>&1 || exit 1 @@ -82,6 +83,14 @@ if [ "$graphflag" != "-k" ]; then cd - fi +echo "$0: loading primary DB" +date +mysql --defaults-group-suffix=tmapadmin -u travmapadmin TravelMapping < TravelMapping-$datestr.sql +/bin/rm $tmwebbase/dbupdating +echo "$0: switching to primary DB" +date +ln -sf $tmwebbase/lib/tm.conf.standard $tmwebbase/lib/tm.conf + echo "$0: installing logs, stats, nmp_merged, graphs, archiving old contents in $tmpdir/$datestr" mkdir -p $tmpdir/$datestr mv $tmwebbase/$logdir $tmpdir/$datestr @@ -95,13 +104,7 @@ if [ "$graphflag" != "-k" ]; then mv $datestr/$graphdir $tmwebbase fi rmdir $datestr -echo "$0: loading primary DB" -date -mysql --defaults-group-suffix=tmapadmin -u travmapadmin TravelMapping < TravelMapping-$datestr.sql -/bin/rm $tmwebbase/dbupdating -echo "$0: switching to primary DB" -date -ln -sf $tmwebbase/lib/tm.conf.standard $tmwebbase/lib/tm.conf + echo "$0: loading DB copy" mysql --defaults-group-suffix=tmapadmin -u travmapadmin TravelMappingCopy < TravelMapping-$datestr.sql echo "$0: moving sql file to archive" diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index b9028f9a..f9d9f9fb 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -77,8 +77,8 @@ def insert(self,w): #print("QTDEBUG: " + str(self) + " insert " + str(w)) if self.points is not None: if self.waypoint_at_same_point(w) is None: + #print("QTDEBUG: " + str(self) + " at " + str(self.unique_locations) + " unique locations") self.unique_locations += 1 - #print("QTDEBUG: " + str(self) + " has " + str(self.unique_locations) + " unique locations") self.points.append(w) if self.unique_locations > 50: # 50 unique points max per quadtree node self.refine() @@ -233,6 +233,15 @@ def total_nodes(self): else: return 1 + self.nw_child.total_nodes() + self.ne_child.total_nodes() + self.sw_child.total_nodes() + self.se_child.total_nodes() + def sort(self): + if self.points is None: + self.ne_child.sort() + self.nw_child.sort() + self.se_child.sort() + self.sw_child.sort() + else: + self.points.sort(key=lambda waypoint: waypoint.route.root + "@" + waypoint.label) + class Waypoint: """This class encapsulates the information about a single waypoint from a .wpt file. @@ -244,7 +253,7 @@ class Waypoint: root is the unique identifier for the route in which this waypoint is defined """ - def __init__(self,line,route): + def __init__(self,line,route,datacheckerrors): """initialize object from a .wpt file line""" self.route = route parts = line.split() @@ -257,8 +266,64 @@ def __init__(self,line,route): self.alt_labels = [] # last has the URL, which needs more work to get lat/lng url_parts = parts[-1].split('=') + if len(url_parts) < 3: + datacheckerrors.append(DatacheckEntry(route,[self.label],'MALFORMED_URL', parts[-1])) + self.lat = 0 + self.lng = 0 + self.colocated = None + self.near_miss_points = None + return lat_string = url_parts[1].split("&")[0] # chop off "&lon" lng_string = url_parts[2].split("&")[0] # chop off possible "&zoom" + + # make sure lat_string is valid + point_count = 0 + for c in range(len(lat_string)): + # check for multiple decimal points + if lat_string[c] == '.': + point_count += 1 + if point_count > 1: + datacheckerrors.append(DatacheckEntry(route,[self.label],'MALFORMED_URL', parts[-1])) + lat_string = "0" + lng_string = "0" + break + # check for minus sign not at beginning + if lat_string[c] == '-' and c > 0: + datacheckerrors.append(DatacheckEntry(route,[self.label],'MALFORMED_URL', parts[-1])) + lat_string = "0" + lng_string = "0" + break + # check for invalid characters + if lat_string[c] not in "-.0123456789": + datacheckerrors.append(DatacheckEntry(route,[self.label],'MALFORMED_URL', parts[-1])) + lat_string = "0" + lng_string = "0" + break + + # make sure lng_string is valid + point_count = 0 + for c in range(len(lng_string)): + # check for multiple decimal points + if lng_string[c] == '.': + point_count += 1 + if point_count > 1: + datacheckerrors.append(DatacheckEntry(route,[self.label],'MALFORMED_URL', parts[-1])) + lat_string = "0" + lng_string = "0" + break + # check for minus sign not at beginning + if lng_string[c] == '-' and c > 0: + datacheckerrors.append(DatacheckEntry(route,[self.label],'MALFORMED_URL', parts[-1])) + lat_string = "0" + lng_string = "0" + break + # check for invalid characters + if lng_string[c] not in "-.0123456789": + datacheckerrors.append(DatacheckEntry(route,[self.label],'MALFORMED_URL', parts[-1])) + lat_string = "0" + lng_string = "0" + break + self.lat = float(lat_string) self.lng = float(lng_string) # also keep track of a list of colocated waypoints, if any @@ -592,8 +657,7 @@ def __init__(self,w1,w2,route): self.segment_name = None def __str__(self): - return self.waypoint1.label + " to " + self.waypoint2.label + \ - " via " + self.route.root + return self.route.readable_name() + " " + self.waypoint1.label + " " + self.waypoint2.label def add_clinched_by(self,traveler): if traveler not in self.clinched_by: @@ -664,7 +728,6 @@ class Route: def __init__(self,line,system,el): """initialize object from a .csv file line, but do not yet read in waypoint file""" - self.line = line fields = line.split(";") if len(fields) != 8: el.add_error("Could not parse csv line: [" + line + @@ -703,9 +766,10 @@ def read_wpt(self,all_waypoints,all_waypoints_lock,datacheckerrors,el,path="../. file.close() w = None for line in lines: - if len(line.rstrip('\n')) > 0: + line = line.strip() + if len(line) > 0: previous_point = w - w = Waypoint(line.rstrip('\n'),self) + w = Waypoint(line,self,datacheckerrors) self.point_list.append(w) # populate unused alt labels for label in w.alt_labels: @@ -792,7 +856,6 @@ class ConnectedRoute: def __init__(self,line,system,el): """initialize the object from the _con.csv line given""" - self.line = line fields = line.split(";") if len(fields) != 5: el.add_error("Could not parse _con.csv line: [" + line + @@ -824,7 +887,7 @@ def __init__(self,line,system,el): rootOrder += 1 if len(self.roots) < 1: el.add_error("No roots in _con.csv line [" + line + "]") - # will be computed for routes in active systems later + # will be computed for routes in active & preview systems later self.mileage = 0.0 def csv_line(self): @@ -921,7 +984,7 @@ class TravelerList: start_waypoint end_waypoint """ - def __init__(self,travelername,systems,route_hash,path="../../../UserData/list_files"): + def __init__(self,travelername,route_hash,path="../../../UserData/list_files"): self.list_entries = [] self.clinched_segments = set() self.traveler_name = travelername[:-5] @@ -1055,7 +1118,7 @@ class DatacheckEntry: code is the error code string, one of SHARP_ANGLE, BAD_ANGLE, DUPLICATE_LABEL, DUPLICATE_COORDS, LABEL_SELFREF, - LABEL_INVALID_CHAR, LONG_SEGMENT, + LABEL_INVALID_CHAR, LONG_SEGMENT, MALFORMED_URL, LABEL_UNDERSCORES, VISIBLE_DISTANCE, LABEL_PARENS, LACKS_GENERIC, BUS_WITH_I, NONTERMINAL_UNDERSCORE, LONG_UNDERSCORE, LABEL_SLASHES, US_BANNER, VISIBLE_HIDDEN_COLOC, @@ -1146,8 +1209,6 @@ def __init__(self,waypoint_list,datacheckerrors): hid_list.append(w) else: vis_list.append(w) - vis_list.sort(key=lambda waypoint: waypoint.route.root + "@" + waypoint.label) - hid_list.sort(key=lambda waypoint: waypoint.route.root + "@" + waypoint.label) datacheckerrors.append(DatacheckEntry(vis_list[0].route,[vis_list[0].label],"VISIBLE_HIDDEN_COLOC", hid_list[0].route.root+"@"+hid_list[0].label)) @@ -1580,8 +1641,9 @@ def __init__(self, all_waypoints, highway_systems, datacheckerrors): vinfo.is_hidden = False continue if len(vinfo.incident_collapsed_edges) > 2: - dc_waypoint = sorted(vinfo.first_waypoint.colocated, key=lambda waypoint: waypoint.route.root + "@" + waypoint.label)[0] - datacheckerrors.append(DatacheckEntry(dc_waypoint.route,[dc_waypoint.label],"HIDDEN_JUNCTION",str(len(vinfo.incident_collapsed_edges)))) + datacheckerrors.append(DatacheckEntry(vinfo.first_waypoint.colocated[0].route, + [vinfo.first_waypoint.colocated[0].label], + "HIDDEN_JUNCTION",str(len(vinfo.incident_collapsed_edges)))) vinfo.is_hidden = False continue # construct from vertex_info this time @@ -2020,12 +2082,12 @@ def __init__(self,filename,descr,vertices,edges,format,category): con_roots = set() for h in highway_systems: - for r in h.con_route_list: - for cr in r.roots: - if cr.root in con_roots: - el.add_error("Duplicate root in con_route lists: " + cr.root) + for cr in h.con_route_list: + for r in cr.roots: + if r.root in con_roots: + el.add_error("Duplicate root in con_route lists: " + r.root) else: - con_roots.add(cr.root) + con_roots.add(r.root) # Make sure every route was listed as a part of some connected route if len(roots) == len(con_roots): @@ -2048,14 +2110,9 @@ def __init__(self,filename,descr,vertices,edges,format,category): # report any duplicate list names as errors if len(duplicate_list_names) > 0: - print("Found " + str(len(duplicate_list_names)) + " DUPLICATE_LIST_NAME cases.") - num_found = 0 - for h in highway_systems: - for r in h.route_list: - if r.region + ' ' + r.list_entry_name() in duplicate_list_names: - el.add_error("Duplicate list name: " + r.region + ' ' + r.list_entry_name()) - num_found += 1 - print("Added " + str(num_found) + " DUPLICATE_LIST_NAME error entries.") + print("Found " + str(len(duplicate_list_names)) + " DUPLICATE_LIST_NAME case(s).") + for d in duplicate_list_names: + el.add_error("Duplicate list name: " + d) else: print("No duplicate list names found.") @@ -2146,6 +2203,14 @@ def run(self): #for h in highway_systems: # read_wpts_for_highway_system(h) +print(et.et() + "Sorting waypoints in Quadtree.") +all_waypoints.sort() + +print(et.et() + "Sorting colocated point lists.") +for w in all_waypoints.point_list(): + if w.colocated is not None: + w.colocated.sort(key=lambda waypoint: waypoint.route.root + "@" + waypoint.label) + print(et.et() + "Finding unprocessed wpt files.", flush=True) unprocessedfile = open(args.logfilepath+'/unprocessedwpts.log','w',encoding='utf-8') if len(all_wpt_files) > 0: @@ -2272,7 +2337,7 @@ def run(self): for t in traveler_ids: if t.endswith('.list'): print(" " + t,end="",flush=True) - traveler_lists.append(TravelerList(t,highway_systems,route_hash,args.userlistfilepath)) + traveler_lists.append(TravelerList(t,route_hash,args.userlistfilepath)) print(" processed " + str(len(traveler_lists)) + " traveler list files.") # Read updates.csv file, just keep in the fields array for now since we're @@ -2364,6 +2429,7 @@ def run(self): s.concurrent.append(other) concurrencyfile.write("New concurrency [" + str(s) + "][" + str(other) + "] (" + str(len(s.concurrent)) + ")\n") else: + other.concurrent = s.concurrent if other not in s.concurrent: s.concurrent.append(other) #concurrencyfile.write("Added concurrency [" + str(s) + "]-[" + str(other) + "] ("+ str(len(s.concurrent)) + ")\n") @@ -2791,6 +2857,7 @@ def run(self): for line in graph_data.waypoint_naming_log: logfile.write(line + '\n') logfile.close() +graph_data.waypoint_naming_log = None # create list of graph information for the DB graph_list = [] @@ -2976,7 +3043,8 @@ def run(self): datacheckfps = [] datacheck_always_error = [ 'DUPLICATE_LABEL', 'HIDDEN_TERMINUS', 'LABEL_INVALID_CHAR', 'LABEL_SLASHES', - 'LONG_UNDERSCORE', 'NONTERMINAL_UNDERSCORE' ] + 'LONG_UNDERSCORE', 'MALFORMED_URL', + 'NONTERMINAL_UNDERSCORE' ] for line in lines: fields = line.rstrip('\n').split(';') if len(fields) != 6: @@ -3163,6 +3231,8 @@ def run(self): print("!", flush=True) print(et.et() + "Found " + str(len(datacheckerrors)) + " datacheck errors.") +datacheckerrors.sort(key=lambda DatacheckEntry: str(DatacheckEntry)) + # now mark false positives print(et.et() + "Marking datacheck false positives.",end="",flush=True) fpfile = open(args.logfilepath+'/nearmatchfps.log','w',encoding='utf-8') @@ -3182,9 +3252,8 @@ def run(self): toremove.append(fp) break if d.match_except_info(fp): - fpfile.write("DCERROR: " + str(d) + "\n") - fpfile.write("FPENTRY: " + fp[0] + ';' + fp[1] + ';' + fp[2] + ';' + fp[3] + ';' + fp[4] + ';' + fp[5] + '\n') - fpfile.write("REPLACEWITH: " + fp[0] + ';' + fp[1] + ';' + fp[2] + ';' + fp[3] + ';' + fp[4] + ';' + d.info + '\n') + fpfile.write("FP_ENTRY: " + fp[0] + ';' + fp[1] + ';' + fp[2] + ';' + fp[3] + ';' + fp[4] + ';' + fp[5] + '\n') + fpfile.write("CHANGETO: " + fp[0] + ';' + fp[1] + ';' + fp[2] + ';' + fp[3] + ';' + fp[4] + ';' + d.info + '\n') fpfile.close() # now remove the ones we matched from the list @@ -3621,6 +3690,6 @@ def run(self): print("Unique locations: " + str(unique_locations)) if args.errorcheck: - print("Data check successful!") + print("!!! DATA CHECK SUCCESSFUL !!!") print("Total run time: " + et.et())