From edf720f4127f1e07b8af4a626416477f042847d9 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Wed, 16 May 2018 22:17:26 -0400 Subject: [PATCH 01/13] HIDDEN_TERMINUS datacheck --- siteupdate/python-teresco/siteupdate.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 57b30a4c..94fc6dfc 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2233,9 +2233,9 @@ def run(self): lines.pop(0) # ignore header line datacheckfps = [] -datacheck_always_error = [ 'DUPLICATE_LABEL', 'LABEL_INVALID_CHAR', - 'LABEL_SLASHES', 'LONG_UNDERSCORE', - 'NONTERMINAL_UNDERSCORE' ] +datacheck_always_error = [ 'DUPLICATE_LABEL', 'HIDDEN_TERMINUS', + 'LABEL_INVALID_CHAR', 'LABEL_SLASHES', + 'LONG_UNDERSCORE', 'NONTERMINAL_UNDERSCORE' ] for line in lines: fields = line.rstrip('\n').split(';') if len(fields) != 6: @@ -2266,6 +2266,17 @@ def run(self): # so the following is simply a placeholder last_visible = None prev_w = None + + # look for hidden termini + if r.point_list[0].is_hidden: + labels = [] + labels.append(r.point_list[0]) + datacheckerrors.append(DatacheckEntry(r,labels,'HIDDEN_TERMINUS')) + if r.point_list[len(r.point_list)-1].is_hidden: + labels = [] + labels.append(r.point_list[len(r.point_list)-1]) + datacheckerrors.append(DatacheckEntry(r,labels,'HIDDEN_TERMINUS')) + for w in r.point_list: # duplicate labels label_list = w.alt_labels.copy() From b5ecd05356f8bc2fdc2c0dcb64dcd6dcb9888914 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Wed, 16 May 2018 22:22:19 -0400 Subject: [PATCH 02/13] namespace after #include --- fpcull/fpcull.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fpcull/fpcull.cpp b/fpcull/fpcull.cpp index 7c66c4a7..eb070c3f 100644 --- a/fpcull/fpcull.cpp +++ b/fpcull/fpcull.cpp @@ -1,7 +1,7 @@ -// Travel Mapping Project, Eric Bryant, 2017 -using namespace std; +// Travel Mapping Project, Eric Bryant, 2017, 2018 #include #include +using namespace std; class entry { public: From c3e2a65fcdf8baf41b40da5655d9abdfa652c43d Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Wed, 16 May 2018 23:42:16 -0400 Subject: [PATCH 03/13] Fix for #74. The datacheck entries need to have labels, not Waypoints. --- siteupdate/python-teresco/siteupdate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 94fc6dfc..c1477a74 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2270,11 +2270,11 @@ def run(self): # look for hidden termini if r.point_list[0].is_hidden: labels = [] - labels.append(r.point_list[0]) + labels.append(r.point_list[0].label) datacheckerrors.append(DatacheckEntry(r,labels,'HIDDEN_TERMINUS')) if r.point_list[len(r.point_list)-1].is_hidden: labels = [] - labels.append(r.point_list[len(r.point_list)-1]) + labels.append(r.point_list[len(r.point_list)-1].label) datacheckerrors.append(DatacheckEntry(r,labels,'HIDDEN_TERMINUS')) for w in r.point_list: From 8d2b18c7ed0d2a7a3e1a00f22fb3a846860e96bc Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Sun, 20 May 2018 08:36:15 -0400 Subject: [PATCH 04/13] Added needed PYTHONIOENCODING variable --- RUNNING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RUNNING.md b/RUNNING.md index 52e066ab..b89d2b94 100644 --- a/RUNNING.md +++ b/RUNNING.md @@ -44,7 +44,7 @@ To do this, you will enter the following at your $ prompt: ``` cd ~/DataProcessing/siteupdate/python-teresco -python3 siteupdate.py -k +PYTHONIOENCODING='utf-8' python3 siteupdate.py -k ``` The process that launches will likely run for several minutes. The "-k" omits graph generation, which is one of the slowest parts of the process. You can leave it out if you want to generate graphs. From edd54b0e8ca133695a74440f275df4726016830a Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Sun, 20 May 2018 09:38:30 -0400 Subject: [PATCH 05/13] mention crtl-c --- RUNNING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/RUNNING.md b/RUNNING.md index b89d2b94..1d0ea361 100644 --- a/RUNNING.md +++ b/RUNNING.md @@ -47,9 +47,9 @@ cd ~/DataProcessing/siteupdate/python-teresco PYTHONIOENCODING='utf-8' python3 siteupdate.py -k ``` -The process that launches will likely run for several minutes. The "-k" omits graph generation, which is one of the slowest parts of the process. You can leave it out if you want to generate graphs. +The process that launches will likely run for several minutes. The "-k" omits graph generation, which is one of the slowest parts of the process. You can leave it out if you want to generate graphs. -If the program runs to completion without reporting errors, you are likely in good shape to make your pull request with your highway data changes. If not, you have things to fix. +If the process gets to the point of reading in user list files, your csv and wpt files are likely good and you might hit "ctrl-c" to kill the process. However, you might wish to let it continue if you have made changes to updates.csv or systemupdates.csv, as those are processed later. Once you are satisfied that your changes do not cause errors, you are in good shape to make your pull request with your highway data changes. If not, you have things to fix. ### Updating before subsequent runs From ff52ec854fb24d0c22132ca47067856affa1bfdf Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Sun, 20 May 2018 19:32:55 -0400 Subject: [PATCH 06/13] -e option for highway data check mode. --- siteupdate/python-teresco/siteupdate.py | 577 ++++++++++++------------ 1 file changed, 298 insertions(+), 279 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index c1477a74..7a10d77d 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1880,6 +1880,7 @@ def __init__(self,filename,descr,vertices,edges,format,category): parser.add_argument("-n", "--nmpmergepath", default="", help="Path to write data with NMPs merged (generated only if specified)") parser.add_argument("-U", "--userlist", default=None, nargs="+", help="For Development: list of users to use in dataset") parser.add_argument("-t", "--numthreads", default="4", help="Number of threads to use for concurrent tasks") +parser.add_argument("-e", "--errorcheck", action="store_true", help="Run only the subset of the process needed to verify highway data changes") args = parser.parse_args() # @@ -2194,7 +2195,7 @@ def run(self): nmpnmp.close() # if requested, rewrite data with near-miss points merged in -if args.nmpmergepath != "": +if args.nmpmergepath != "" and not args.errorcheck: print(et.et() + "Writing near-miss point merged wpt files.", flush=True) for h in highway_systems: print(h.systemname, end="", flush=True) @@ -2998,7 +2999,7 @@ def run(self): # Build a graph structure out of all highway data in active and # preview systems -if args.skipgraphs: +if args.skipgraphs or args.errorcheck: print(et.et() + "SKIPPING generation of graphs.", flush=True) else: print(et.et() + "Setting up for graphs of highway data.", flush=True) @@ -3177,338 +3178,356 @@ def run(self): graph_types.append(['continent', 'Routes Within a Continent', 'These graphs contain the routes on a continent.']) print("!") + +# datacheck.log file +print(et.et() + "Writing datacheck.log") +logfile = open(args.logfilepath + '/datacheck.log', 'w') +logfile.write("Log file created at: " + str(datetime.datetime.now()) + "\n") +logfile.write("Datacheck errors that have been flagged as false positives are not included.\n") +if len(datacheckerrors) > 0: + for d in datacheckerrors: + if not d.fp: + logfile.write(str(d.route.root)) + for l in d.labels: + logfile.write(" " + l) + logfile.write(" "+d.code+" "+d.info+"\n") +else: + logfile.write("No datacheck errors found.") +logfile.close() +if args.errorcheck: + print(et.et() + "SKIPPING database file.") +else: + print(et.et() + "Writing database file " + args.databasename + ".sql.") + # Once all data is read in and processed, create a .sql file that will + # create all of the DB tables to be used by other parts of the project + sqlfile = open(args.databasename+'.sql','w',encoding='UTF-8') + # Note: removed "USE" line, DB name must be specified on the mysql command line -print(et.et() + "Writing database file " + args.databasename + ".sql.") -# Once all data is read in and processed, create a .sql file that will -# create all of the DB tables to be used by other parts of the project -sqlfile = open(args.databasename+'.sql','w',encoding='UTF-8') -# Note: removed "USE" line, DB name must be specified on the mysql command line - -# we have to drop tables in the right order to avoid foreign key errors -sqlfile.write('DROP TABLE IF EXISTS datacheckErrors;\n') -sqlfile.write('DROP TABLE IF EXISTS clinchedConnectedRoutes;\n') -sqlfile.write('DROP TABLE IF EXISTS clinchedRoutes;\n') -sqlfile.write('DROP TABLE IF EXISTS clinchedOverallMileageByRegion;\n') -sqlfile.write('DROP TABLE IF EXISTS clinchedSystemMileageByRegion;\n') -sqlfile.write('DROP TABLE IF EXISTS overallMileageByRegion;\n') -sqlfile.write('DROP TABLE IF EXISTS systemMileageByRegion;\n') -sqlfile.write('DROP TABLE IF EXISTS clinched;\n') -sqlfile.write('DROP TABLE IF EXISTS segments;\n') -sqlfile.write('DROP TABLE IF EXISTS waypoints;\n') -sqlfile.write('DROP TABLE IF EXISTS connectedRouteRoots;\n') -sqlfile.write('DROP TABLE IF EXISTS connectedRoutes;\n') -sqlfile.write('DROP TABLE IF EXISTS routes;\n') -sqlfile.write('DROP TABLE IF EXISTS systems;\n') -sqlfile.write('DROP TABLE IF EXISTS updates;\n') -sqlfile.write('DROP TABLE IF EXISTS systemUpdates;\n') -sqlfile.write('DROP TABLE IF EXISTS regions;\n') -sqlfile.write('DROP TABLE IF EXISTS countries;\n') -sqlfile.write('DROP TABLE IF EXISTS continents;\n') - -# first, continents, countries, and regions -sqlfile.write('CREATE TABLE continents (code VARCHAR(3), name VARCHAR(15), PRIMARY KEY(code));\n') -sqlfile.write('INSERT INTO continents VALUES\n') -first = True -for c in continents: - if not first: - sqlfile.write(",") - first = False + # we have to drop tables in the right order to avoid foreign key errors + sqlfile.write('DROP TABLE IF EXISTS datacheckErrors;\n') + sqlfile.write('DROP TABLE IF EXISTS clinchedConnectedRoutes;\n') + sqlfile.write('DROP TABLE IF EXISTS clinchedRoutes;\n') + sqlfile.write('DROP TABLE IF EXISTS clinchedOverallMileageByRegion;\n') + sqlfile.write('DROP TABLE IF EXISTS clinchedSystemMileageByRegion;\n') + sqlfile.write('DROP TABLE IF EXISTS overallMileageByRegion;\n') + sqlfile.write('DROP TABLE IF EXISTS systemMileageByRegion;\n') + sqlfile.write('DROP TABLE IF EXISTS clinched;\n') + sqlfile.write('DROP TABLE IF EXISTS segments;\n') + sqlfile.write('DROP TABLE IF EXISTS waypoints;\n') + sqlfile.write('DROP TABLE IF EXISTS connectedRouteRoots;\n') + sqlfile.write('DROP TABLE IF EXISTS connectedRoutes;\n') + sqlfile.write('DROP TABLE IF EXISTS routes;\n') + sqlfile.write('DROP TABLE IF EXISTS systems;\n') + sqlfile.write('DROP TABLE IF EXISTS updates;\n') + sqlfile.write('DROP TABLE IF EXISTS systemUpdates;\n') + sqlfile.write('DROP TABLE IF EXISTS regions;\n') + sqlfile.write('DROP TABLE IF EXISTS countries;\n') + sqlfile.write('DROP TABLE IF EXISTS continents;\n') + + # first, continents, countries, and regions + sqlfile.write('CREATE TABLE continents (code VARCHAR(3), name VARCHAR(15), PRIMARY KEY(code));\n') + sqlfile.write('INSERT INTO continents VALUES\n') + first = True + for c in continents: + if not first: + sqlfile.write(",") + first = False sqlfile.write("('" + c[0] + "','" + c[1] + "')\n") -sqlfile.write(";\n") - -sqlfile.write('CREATE TABLE countries (code VARCHAR(3), name VARCHAR(32), PRIMARY KEY(code));\n') -sqlfile.write('INSERT INTO countries VALUES\n') -first = True -for c in countries: - if not first: - sqlfile.write(",") - first = False + sqlfile.write(";\n") + + sqlfile.write('CREATE TABLE countries (code VARCHAR(3), name VARCHAR(32), PRIMARY KEY(code));\n') + sqlfile.write('INSERT INTO countries VALUES\n') + first = True + for c in countries: + if not first: + sqlfile.write(",") + first = False sqlfile.write("('" + c[0] + "','" + c[1].replace("'","''") + "')\n") -sqlfile.write(";\n") - -sqlfile.write('CREATE TABLE regions (code VARCHAR(8), name VARCHAR(48), country VARCHAR(3), continent VARCHAR(3), regiontype VARCHAR(32), PRIMARY KEY(code), FOREIGN KEY (country) REFERENCES countries(code), FOREIGN KEY (continent) REFERENCES continents(code));\n') -sqlfile.write('INSERT INTO regions VALUES\n') -first = True -for r in all_regions: - if not first: - sqlfile.write(",") - first = False - sqlfile.write("('" + r[0] + "','" + r[1].replace("'","''") + "','" + r[2] + "','" + r[3] + "','" + r[4] + "')\n") -sqlfile.write(";\n") - -# next, a table of the systems, consisting of the system name in the -# field 'name', the system's country code, its full name, the default -# color for its mapping, a level (one of active, preview, devel), and -# a boolean indicating if the system is active for mapping in the -# project in the field 'active' -sqlfile.write('CREATE TABLE systems (systemName VARCHAR(10), countryCode CHAR(3), fullName VARCHAR(60), color VARCHAR(16), level VARCHAR(10), tier INTEGER, PRIMARY KEY(systemName));\n') -sqlfile.write('INSERT INTO systems VALUES\n') -first = True -for h in highway_systems: - if not first: - sqlfile.write(",") - first = False - sqlfile.write("('" + h.systemname + "','" + h.country + "','" + - h.fullname + "','" + h.color + "','" + h.level + - "','" + str(h.tier) + "')\n") -sqlfile.write(";\n") - -# next, a table of highways, with the same fields as in the first line -sqlfile.write('CREATE TABLE routes (systemName VARCHAR(10), region VARCHAR(8), route VARCHAR(16), banner VARCHAR(6), abbrev VARCHAR(3), city VARCHAR(100), root VARCHAR(32), mileage FLOAT, PRIMARY KEY(root), FOREIGN KEY (systemName) REFERENCES systems(systemName));\n') -sqlfile.write('INSERT INTO routes VALUES\n') -first = True -for h in highway_systems: - for r in h.route_list: + sqlfile.write(";\n") + + sqlfile.write('CREATE TABLE regions (code VARCHAR(8), name VARCHAR(48), country VARCHAR(3), continent VARCHAR(3), regiontype VARCHAR(32), PRIMARY KEY(code), FOREIGN KEY (country) REFERENCES countries(code), FOREIGN KEY (continent) REFERENCES continents(code));\n') + sqlfile.write('INSERT INTO regions VALUES\n') + first = True + for r in all_regions: if not first: sqlfile.write(",") first = False - sqlfile.write("(" + r.csv_line() + ")\n") -sqlfile.write(";\n") + sqlfile.write("('" + r[0] + "','" + r[1].replace("'","''") + "','" + r[2] + "','" + r[3] + "','" + r[4] + "')\n") + sqlfile.write(";\n") -# connected routes table, but only first "root" in each in this table -sqlfile.write('CREATE TABLE connectedRoutes (systemName VARCHAR(10), route VARCHAR(16), banner VARCHAR(6), groupName VARCHAR(100), firstRoot VARCHAR(32), mileage FLOAT, PRIMARY KEY(firstRoot), FOREIGN KEY (firstRoot) REFERENCES routes(root));\n') -sqlfile.write('INSERT INTO connectedRoutes VALUES\n') -first = True -for h in highway_systems: - for cr in h.con_route_list: + # next, a table of the systems, consisting of the system name in the + # field 'name', the system's country code, its full name, the default + # color for its mapping, a level (one of active, preview, devel), and + # a boolean indicating if the system is active for mapping in the + # project in the field 'active' + sqlfile.write('CREATE TABLE systems (systemName VARCHAR(10), countryCode CHAR(3), fullName VARCHAR(60), color VARCHAR(16), level VARCHAR(10), tier INTEGER, PRIMARY KEY(systemName));\n') + sqlfile.write('INSERT INTO systems VALUES\n') + first = True + for h in highway_systems: if not first: sqlfile.write(",") first = False - sqlfile.write("(" + cr.csv_line() + ")\n") -sqlfile.write(";\n") + sqlfile.write("('" + h.systemname + "','" + h.country + "','" + + h.fullname + "','" + h.color + "','" + h.level + + "','" + str(h.tier) + "')\n") + sqlfile.write(";\n") -# This table has remaining roots for any connected route -# that connects multiple routes/roots -sqlfile.write('CREATE TABLE connectedRouteRoots (firstRoot VARCHAR(32), root VARCHAR(32), FOREIGN KEY (firstRoot) REFERENCES connectedRoutes(firstRoot));\n') -first = True -for h in highway_systems: - for cr in h.con_route_list: - if len(cr.roots) > 1: - for i in range(1,len(cr.roots)): - if first: - sqlfile.write('INSERT INTO connectedRouteRoots VALUES\n') - if not first: - sqlfile.write(",") - first = False - sqlfile.write("('" + cr.roots[0].root + "','" + cr.roots[i].root + "')\n") -sqlfile.write(";\n") + # next, a table of highways, with the same fields as in the first line + sqlfile.write('CREATE TABLE routes (systemName VARCHAR(10), region VARCHAR(8), route VARCHAR(16), banner VARCHAR(6), abbrev VARCHAR(3), city VARCHAR(100), root VARCHAR(32), mileage FLOAT, PRIMARY KEY(root), FOREIGN KEY (systemName) REFERENCES systems(systemName));\n') + sqlfile.write('INSERT INTO routes VALUES\n') + first = True + for h in highway_systems: + for r in h.route_list: + if not first: + sqlfile.write(",") + first = False + sqlfile.write("(" + r.csv_line() + ")\n") + sqlfile.write(";\n") -# Now, a table with raw highway route data: list of points, in order, that define the route -sqlfile.write('CREATE TABLE waypoints (pointId INTEGER, pointName VARCHAR(20), latitude DOUBLE, longitude DOUBLE, root VARCHAR(32), PRIMARY KEY(pointId), FOREIGN KEY (root) REFERENCES routes(root));\n') -point_num = 0 -for h in highway_systems: - for r in h.route_list: - sqlfile.write('INSERT INTO waypoints VALUES\n') - first = True - for w in r.point_list: + # connected routes table, but only first "root" in each in this table + sqlfile.write('CREATE TABLE connectedRoutes (systemName VARCHAR(10), route VARCHAR(16), banner VARCHAR(6), groupName VARCHAR(100), firstRoot VARCHAR(32), mileage FLOAT, PRIMARY KEY(firstRoot), FOREIGN KEY (firstRoot) REFERENCES routes(root));\n') + sqlfile.write('INSERT INTO connectedRoutes VALUES\n') + first = True + for h in highway_systems: + for cr in h.con_route_list: if not first: sqlfile.write(",") first = False - w.point_num = point_num - sqlfile.write("(" + w.csv_line(point_num) + ")\n") - point_num+=1 - sqlfile.write(";\n") + sqlfile.write("(" + cr.csv_line() + ")\n") + sqlfile.write(";\n") -# Build indices to speed latitude/longitude joins for intersecting highway queries -sqlfile.write('CREATE INDEX `latitude` ON waypoints(`latitude`);\n') -sqlfile.write('CREATE INDEX `longitude` ON waypoints(`longitude`);\n') + # This table has remaining roots for any connected route + # that connects multiple routes/roots + sqlfile.write('CREATE TABLE connectedRouteRoots (firstRoot VARCHAR(32), root VARCHAR(32), FOREIGN KEY (firstRoot) REFERENCES connectedRoutes(firstRoot));\n') + first = True + for h in highway_systems: + for cr in h.con_route_list: + if len(cr.roots) > 1: + for i in range(1,len(cr.roots)): + if first: + sqlfile.write('INSERT INTO connectedRouteRoots VALUES\n') + if not first: + sqlfile.write(",") + first = False + sqlfile.write("('" + cr.roots[0].root + "','" + cr.roots[i].root + "')\n") + sqlfile.write(";\n") -# Table of all HighwaySegments. -sqlfile.write('CREATE TABLE segments (segmentId INTEGER, waypoint1 INTEGER, waypoint2 INTEGER, root VARCHAR(32), PRIMARY KEY (segmentId), FOREIGN KEY (waypoint1) REFERENCES waypoints(pointId), FOREIGN KEY (waypoint2) REFERENCES waypoints(pointId), FOREIGN KEY (root) REFERENCES routes(root));\n') -segment_num = 0 -clinched_list = [] -for h in highway_systems: - for r in h.route_list: - sqlfile.write('INSERT INTO segments VALUES\n') + # Now, a table with raw highway route data: list of points, in order, that define the route + sqlfile.write('CREATE TABLE waypoints (pointId INTEGER, pointName VARCHAR(20), latitude DOUBLE, longitude DOUBLE, root VARCHAR(32), PRIMARY KEY(pointId), FOREIGN KEY (root) REFERENCES routes(root));\n') + point_num = 0 + for h in highway_systems: + for r in h.route_list: + sqlfile.write('INSERT INTO waypoints VALUES\n') + first = True + for w in r.point_list: + if not first: + sqlfile.write(",") + first = False + w.point_num = point_num + sqlfile.write("(" + w.csv_line(point_num) + ")\n") + point_num+=1 + sqlfile.write(";\n") + + # Build indices to speed latitude/longitude joins for intersecting highway queries + sqlfile.write('CREATE INDEX `latitude` ON waypoints(`latitude`);\n') + sqlfile.write('CREATE INDEX `longitude` ON waypoints(`longitude`);\n') + + # Table of all HighwaySegments. + sqlfile.write('CREATE TABLE segments (segmentId INTEGER, waypoint1 INTEGER, waypoint2 INTEGER, root VARCHAR(32), PRIMARY KEY (segmentId), FOREIGN KEY (waypoint1) REFERENCES waypoints(pointId), FOREIGN KEY (waypoint2) REFERENCES waypoints(pointId), FOREIGN KEY (root) REFERENCES routes(root));\n') + segment_num = 0 + clinched_list = [] + for h in highway_systems: + for r in h.route_list: + sqlfile.write('INSERT INTO segments VALUES\n') + first = True + for s in r.segment_list: + if not first: + sqlfile.write(",") + first = False + sqlfile.write("(" + s.csv_line(segment_num) + ")\n") + for t in s.clinched_by: + clinched_list.append("'" + str(segment_num) + "','" + t.traveler_name + "'") + segment_num += 1 + sqlfile.write(";\n") + + # maybe a separate traveler table will make sense but for now, I'll just use + # the name from the .list name + sqlfile.write('CREATE TABLE clinched (segmentId INTEGER, traveler VARCHAR(48), FOREIGN KEY (segmentId) REFERENCES segments(segmentId));\n') + for start in range(0, len(clinched_list), 10000): + sqlfile.write('INSERT INTO clinched VALUES\n') first = True - for s in r.segment_list: + for c in clinched_list[start:start+10000]: if not first: sqlfile.write(",") first = False - sqlfile.write("(" + s.csv_line(segment_num) + ")\n") - for t in s.clinched_by: - clinched_list.append("'" + str(segment_num) + "','" + t.traveler_name + "'") - segment_num += 1 + sqlfile.write("(" + c + ")\n") sqlfile.write(";\n") - -# maybe a separate traveler table will make sense but for now, I'll just use -# the name from the .list name -sqlfile.write('CREATE TABLE clinched (segmentId INTEGER, traveler VARCHAR(48), FOREIGN KEY (segmentId) REFERENCES segments(segmentId));\n') -for start in range(0, len(clinched_list), 10000): - sqlfile.write('INSERT INTO clinched VALUES\n') + + # overall mileage by region data (with concurrencies accounted for, + # active systems only then active+preview) + sqlfile.write('CREATE TABLE overallMileageByRegion (region VARCHAR(8), activeMileage FLOAT, activePreviewMileage FLOAT);\n') + sqlfile.write('INSERT INTO overallMileageByRegion VALUES\n') first = True - for c in clinched_list[start:start+10000]: + for region in list(active_preview_mileage_by_region.keys()): if not first: sqlfile.write(",") first = False - sqlfile.write("(" + c + ")\n") + active_only_mileage = 0.0 + active_preview_mileage = 0.0 + if region in list(active_only_mileage_by_region.keys()): + active_only_mileage = active_only_mileage_by_region[region] + if region in list(active_preview_mileage_by_region.keys()): + active_preview_mileage = active_preview_mileage_by_region[region] + sqlfile.write("('" + region + "','" + + str(active_only_mileage) + "','" + + str(active_preview_mileage) + "')\n") sqlfile.write(";\n") -# overall mileage by region data (with concurrencies accounted for, -# active systems only then active+preview) -sqlfile.write('CREATE TABLE overallMileageByRegion (region VARCHAR(8), activeMileage FLOAT, activePreviewMileage FLOAT);\n') -sqlfile.write('INSERT INTO overallMileageByRegion VALUES\n') -first = True -for region in list(active_preview_mileage_by_region.keys()): - if not first: - sqlfile.write(",") - first = False - active_only_mileage = 0.0 - active_preview_mileage = 0.0 - if region in list(active_only_mileage_by_region.keys()): - active_only_mileage = active_only_mileage_by_region[region] - if region in list(active_preview_mileage_by_region.keys()): - active_preview_mileage = active_preview_mileage_by_region[region] - sqlfile.write("('" + region + "','" + - str(active_only_mileage) + "','" + - str(active_preview_mileage) + "')\n") -sqlfile.write(";\n") - -# system mileage by region data (with concurrencies accounted for, -# active systems and preview systems only) -sqlfile.write('CREATE TABLE systemMileageByRegion (systemName VARCHAR(10), region VARCHAR(8), mileage FLOAT, FOREIGN KEY (systemName) REFERENCES systems(systemName));\n') -sqlfile.write('INSERT INTO systemMileageByRegion VALUES\n') -first = True -for h in highway_systems: - if h.active_or_preview(): - for region in list(h.mileage_by_region.keys()): + # system mileage by region data (with concurrencies accounted for, + # active systems and preview systems only) + sqlfile.write('CREATE TABLE systemMileageByRegion (systemName VARCHAR(10), region VARCHAR(8), mileage FLOAT, FOREIGN KEY (systemName) REFERENCES systems(systemName));\n') + sqlfile.write('INSERT INTO systemMileageByRegion VALUES\n') + first = True + for h in highway_systems: + if h.active_or_preview(): + for region in list(h.mileage_by_region.keys()): + if not first: + sqlfile.write(",") + first = False + sqlfile.write("('" + h.systemname + "','" + region + "','" + str(h.mileage_by_region[region]) + "')\n") + sqlfile.write(";\n") + + # clinched overall mileage by region data (with concurrencies + # accounted for, active systems and preview systems only) + sqlfile.write('CREATE TABLE clinchedOverallMileageByRegion (region VARCHAR(8), traveler VARCHAR(48), activeMileage FLOAT, activePreviewMileage FLOAT);\n') + sqlfile.write('INSERT INTO clinchedOverallMileageByRegion VALUES\n') + first = True + for t in traveler_lists: + for region in list(t.active_preview_mileage_by_region.keys()): if not first: sqlfile.write(",") first = False - sqlfile.write("('" + h.systemname + "','" + region + "','" + str(h.mileage_by_region[region]) + "')\n") -sqlfile.write(";\n") - -# clinched overall mileage by region data (with concurrencies -# accounted for, active systems and preview systems only) -sqlfile.write('CREATE TABLE clinchedOverallMileageByRegion (region VARCHAR(8), traveler VARCHAR(48), activeMileage FLOAT, activePreviewMileage FLOAT);\n') -sqlfile.write('INSERT INTO clinchedOverallMileageByRegion VALUES\n') -first = True -for t in traveler_lists: - for region in list(t.active_preview_mileage_by_region.keys()): - if not first: - sqlfile.write(",") - first = False - active_miles = 0.0 - if region in list(t.active_only_mileage_by_region.keys()): - active_miles = t.active_only_mileage_by_region[region] - sqlfile.write("('" + region + "','" + t.traveler_name + "','" + - str(active_miles) + "','" + - str(t.active_preview_mileage_by_region[region]) + "')\n") -sqlfile.write(";\n") - -# clinched system mileage by region data (with concurrencies accounted -# for, active systems and preview systems only) -sqlfile.write('CREATE TABLE clinchedSystemMileageByRegion (systemName VARCHAR(10), region VARCHAR(8), traveler VARCHAR(48), mileage FLOAT, FOREIGN KEY (systemName) REFERENCES systems(systemName));\n') -sqlfile.write('INSERT INTO clinchedSystemMileageByRegion VALUES\n') -first = True -for line in csmbr_values: - if not first: - sqlfile.write(",") - first = False - sqlfile.write(line + "\n") -sqlfile.write(";\n") - -# clinched mileage by connected route, active systems and preview -# systems only -sqlfile.write('CREATE TABLE clinchedConnectedRoutes (route VARCHAR(32), traveler VARCHAR(48), mileage FLOAT, clinched BOOLEAN, FOREIGN KEY (route) REFERENCES connectedRoutes(firstRoot));\n') -for start in range(0, len(ccr_values), 10000): - sqlfile.write('INSERT INTO clinchedConnectedRoutes VALUES\n') + active_miles = 0.0 + if region in list(t.active_only_mileage_by_region.keys()): + active_miles = t.active_only_mileage_by_region[region] + sqlfile.write("('" + region + "','" + t.traveler_name + "','" + + str(active_miles) + "','" + + str(t.active_preview_mileage_by_region[region]) + "')\n") + sqlfile.write(";\n") + + # clinched system mileage by region data (with concurrencies accounted + # for, active systems and preview systems only) + sqlfile.write('CREATE TABLE clinchedSystemMileageByRegion (systemName VARCHAR(10), region VARCHAR(8), traveler VARCHAR(48), mileage FLOAT, FOREIGN KEY (systemName) REFERENCES systems(systemName));\n') + sqlfile.write('INSERT INTO clinchedSystemMileageByRegion VALUES\n') first = True - for line in ccr_values[start:start+10000]: + for line in csmbr_values: if not first: sqlfile.write(",") first = False sqlfile.write(line + "\n") sqlfile.write(";\n") -# clinched mileage by route, active systems and preview systems only -sqlfile.write('CREATE TABLE clinchedRoutes (route VARCHAR(32), traveler VARCHAR(48), mileage FLOAT, clinched BOOLEAN, FOREIGN KEY (route) REFERENCES routes(root));\n') -for start in range(0, len(cr_values), 10000): - sqlfile.write('INSERT INTO clinchedRoutes VALUES\n') + # clinched mileage by connected route, active systems and preview + # systems only + sqlfile.write('CREATE TABLE clinchedConnectedRoutes (route VARCHAR(32), traveler VARCHAR(48), mileage FLOAT, clinched BOOLEAN, FOREIGN KEY (route) REFERENCES connectedRoutes(firstRoot));\n') + for start in range(0, len(ccr_values), 10000): + sqlfile.write('INSERT INTO clinchedConnectedRoutes VALUES\n') + first = True + for line in ccr_values[start:start+10000]: + if not first: + sqlfile.write(",") + first = False + sqlfile.write(line + "\n") + sqlfile.write(";\n") + + # clinched mileage by route, active systems and preview systems only + sqlfile.write('CREATE TABLE clinchedRoutes (route VARCHAR(32), traveler VARCHAR(48), mileage FLOAT, clinched BOOLEAN, FOREIGN KEY (route) REFERENCES routes(root));\n') + for start in range(0, len(cr_values), 10000): + sqlfile.write('INSERT INTO clinchedRoutes VALUES\n') + first = True + for line in cr_values[start:start+10000]: + if not first: + sqlfile.write(",") + first = False + sqlfile.write(line + "\n") + sqlfile.write(";\n") + + # updates entries + sqlfile.write('CREATE TABLE updates (date VARCHAR(10), region VARCHAR(60), route VARCHAR(80), root VARCHAR(32), description VARCHAR(1024));\n') + sqlfile.write('INSERT INTO updates VALUES\n') first = True - for line in cr_values[start:start+10000]: + for update in updates: if not first: sqlfile.write(",") first = False - sqlfile.write(line + "\n") + sqlfile.write("('"+update[0]+"','"+update[1].replace("'","''")+"','"+update[2].replace("'","''")+"','"+update[3]+"','"+update[4].replace("'","''")+"')\n") sqlfile.write(";\n") -# updates entries -sqlfile.write('CREATE TABLE updates (date VARCHAR(10), region VARCHAR(60), route VARCHAR(80), root VARCHAR(32), description VARCHAR(1024));\n') -sqlfile.write('INSERT INTO updates VALUES\n') -first = True -for update in updates: - if not first: - sqlfile.write(",") - first = False - sqlfile.write("('"+update[0]+"','"+update[1].replace("'","''")+"','"+update[2].replace("'","''")+"','"+update[3]+"','"+update[4].replace("'","''")+"')\n") -sqlfile.write(";\n") - -# systemUpdates entries -sqlfile.write('CREATE TABLE systemUpdates (date VARCHAR(10), region VARCHAR(48), systemName VARCHAR(10), description VARCHAR(128), statusChange VARCHAR(16));\n') -sqlfile.write('INSERT INTO systemUpdates VALUES\n') -first = True -for systemupdate in systemupdates: - if not first: - sqlfile.write(",") - first = False - sqlfile.write("('"+systemupdate[0]+"','"+systemupdate[1].replace("'","''")+"','"+systemupdate[2]+"','"+systemupdate[3].replace("'","''")+"','"+systemupdate[4]+"')\n") -sqlfile.write(";\n") - -# datacheck errors into the db -sqlfile.write('CREATE TABLE datacheckErrors (route VARCHAR(32), label1 VARCHAR(50), label2 VARCHAR(20), label3 VARCHAR(20), code VARCHAR(20), value VARCHAR(32), falsePositive BOOLEAN, FOREIGN KEY (route) REFERENCES routes(root));\n') -if len(datacheckerrors) > 0: - sqlfile.write('INSERT INTO datacheckErrors VALUES\n') + # systemUpdates entries + sqlfile.write('CREATE TABLE systemUpdates (date VARCHAR(10), region VARCHAR(48), systemName VARCHAR(10), description VARCHAR(128), statusChange VARCHAR(16));\n') + sqlfile.write('INSERT INTO systemUpdates VALUES\n') first = True - for d in datacheckerrors: + for systemupdate in systemupdates: if not first: - sqlfile.write(',') + sqlfile.write(",") first = False - sqlfile.write("('"+str(d.route.root)+"',") - if len(d.labels) == 0: - sqlfile.write("'','','',") - elif len(d.labels) == 1: - sqlfile.write("'"+d.labels[0]+"','','',") - elif len(d.labels) == 2: - sqlfile.write("'"+d.labels[0]+"','"+d.labels[1]+"','',") - else: - sqlfile.write("'"+d.labels[0]+"','"+d.labels[1]+"','"+d.labels[2]+"',") - if d.fp: - fp = '1' - else: - fp = '0' - sqlfile.write("'"+d.code+"','"+d.info+"','"+fp+"')\n") -sqlfile.write(";\n") - -# update graph info in DB if graphs were generated -if not args.skipgraphs: - sqlfile.write('DROP TABLE IF EXISTS graphs;\n') - sqlfile.write('DROP TABLE IF EXISTS graphTypes;\n') - sqlfile.write('CREATE TABLE graphTypes (category VARCHAR(12), descr VARCHAR(100), longDescr TEXT, PRIMARY KEY(category));\n') - if len(graph_types) > 0: - sqlfile.write('INSERT INTO graphTypes VALUES\n') - first = True - for g in graph_types: - if not first: - sqlfile.write(',') - first = False - sqlfile.write("('" + g[0] + "','" + g[1] + "','" + g[2] + "')\n") - sqlfile.write(";\n") + sqlfile.write("('"+systemupdate[0]+"','"+systemupdate[1].replace("'","''")+"','"+systemupdate[2]+"','"+systemupdate[3].replace("'","''")+"','"+systemupdate[4]+"')\n") + sqlfile.write(";\n") - sqlfile.write('CREATE TABLE graphs (filename VARCHAR(32), descr VARCHAR(100), vertices INTEGER, edges INTEGER, format VARCHAR(10), category VARCHAR(12), FOREIGN KEY (category) REFERENCES graphTypes(category));\n') - if len(graph_list) > 0: - sqlfile.write('INSERT INTO graphs VALUES\n') + # datacheck errors into the db + sqlfile.write('CREATE TABLE datacheckErrors (route VARCHAR(32), label1 VARCHAR(50), label2 VARCHAR(20), label3 VARCHAR(20), code VARCHAR(20), value VARCHAR(32), falsePositive BOOLEAN, FOREIGN KEY (route) REFERENCES routes(root));\n') + if len(datacheckerrors) > 0: + sqlfile.write('INSERT INTO datacheckErrors VALUES\n') first = True - for g in graph_list: + for d in datacheckerrors: if not first: sqlfile.write(',') first = False - sqlfile.write("('" + g.filename + "','" + g.descr.replace("'","''") + "','" + str(g.vertices) + "','" + str(g.edges) + "','" + g.format + "','" + g.category + "')\n") - sqlfile.write(";\n") + sqlfile.write("('"+str(d.route.root)+"',") + if len(d.labels) == 0: + sqlfile.write("'','','',") + elif len(d.labels) == 1: + sqlfile.write("'"+d.labels[0]+"','','',") + elif len(d.labels) == 2: + sqlfile.write("'"+d.labels[0]+"','"+d.labels[1]+"','',") + else: + sqlfile.write("'"+d.labels[0]+"','"+d.labels[1]+"','"+d.labels[2]+"',") + if d.fp: + fp = '1' + else: + fp = '0' + sqlfile.write("'"+d.code+"','"+d.info+"','"+fp+"')\n") + sqlfile.write(";\n") + + # update graph info in DB if graphs were generated + if not args.skipgraphs: + sqlfile.write('DROP TABLE IF EXISTS graphs;\n') + sqlfile.write('DROP TABLE IF EXISTS graphTypes;\n') + sqlfile.write('CREATE TABLE graphTypes (category VARCHAR(12), descr VARCHAR(100), longDescr TEXT, PRIMARY KEY(category));\n') + if len(graph_types) > 0: + sqlfile.write('INSERT INTO graphTypes VALUES\n') + first = True + for g in graph_types: + if not first: + sqlfile.write(',') + first = False + sqlfile.write("('" + g[0] + "','" + g[1] + "','" + g[2] + "')\n") + sqlfile.write(";\n") + + sqlfile.write('CREATE TABLE graphs (filename VARCHAR(32), descr VARCHAR(100), vertices INTEGER, edges INTEGER, format VARCHAR(10), category VARCHAR(12), FOREIGN KEY (category) REFERENCES graphTypes(category));\n') + if len(graph_list) > 0: + sqlfile.write('INSERT INTO graphs VALUES\n') + first = True + for g in graph_list: + if not first: + sqlfile.write(',') + first = False + sqlfile.write("('" + g.filename + "','" + g.descr.replace("'","''") + "','" + str(g.vertices) + "','" + str(g.edges) + "','" + g.format + "','" + g.category + "')\n") + sqlfile.write(";\n") -sqlfile.close() + sqlfile.close() # print some statistics print(et.et() + "Processed " + str(len(highway_systems)) + " highway systems.") From adeccd57b4876231f54fe11b0e25096b43f62e75 Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Sun, 20 May 2018 19:33:09 -0400 Subject: [PATCH 07/13] Script to run siteupdate.py in highway data check mode. --- siteupdate/python-teresco/datacheck.sh | 40 ++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 siteupdate/python-teresco/datacheck.sh diff --git a/siteupdate/python-teresco/datacheck.sh b/siteupdate/python-teresco/datacheck.sh new file mode 100644 index 00000000..06372402 --- /dev/null +++ b/siteupdate/python-teresco/datacheck.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# +# script to run check-only site update on local data +# +# Jim Teresco, Sun May 20 16:27:18 EDT 2018 +# +set -e +install=1 +pull=1 +if [ -d "$HOME/HighwayData" ]; then + tmbase=$HOME +elif [ -d "$HOME/travelmapping/HighwayData" ]; then + tmbase=$HOME/travelmapping +else + echo "$0: could not find HighwayData repository" + exit 1 +fi +logdir=logs +statdir=stats +date +# process command line args +for arg in "$@"; do + if [ "$arg" == "--nopull" ]; then + pull=0 + fi + shift +done +if [ "$pull" == "1" ]; then + echo "$0: updating TM repositories" + (cd $tmbase/HighwayData; git pull) + (cd $tmbase/UserData; git pull) +fi + +echo "$0: creating directories" +mkdir -p $logdir $statdir + +echo "$0: launching siteupdate.py" +PYTHONIOENCODING='utf-8' ./siteupdate.py -e -l $logdir -c $statdir | tee $logdir/siteupdate.log 2>&1 || exit 1 +date +echo "$0: complete" From 62c6f003e0c14b06385d240fb58b50ae4b9bc288 Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Sun, 20 May 2018 19:44:08 -0400 Subject: [PATCH 08/13] Update instructions for new mode. #73 --- RUNNING.md | 36 +++++++++++------------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/RUNNING.md b/RUNNING.md index 52e066ab..93263b84 100644 --- a/RUNNING.md +++ b/RUNNING.md @@ -1,4 +1,4 @@ -This document describes how to use the site update code on the Travel Mapping server to test your changes to highway data before submitting a pull request. The idea is to speed the process of getting your changes and updates into the site by finding things like missing or misnamed files, missing/erroneous/duplicate csv entries, a WPT file with malformed lines, mismatches between .csv and _con.csv files, misspelled CSV entries or file names, etc, etc, etc. By testing before submitting the pull request, problems could be caught by those making the changes rather than having to wait for the next "official" site update to find out. +This document describes how to use the site update code on the Travel Mapping server to test your changes to highway data before submitting a pull request. The idea is to speed the process of getting your changes and updates into the site by finding things like missing or misnamed files, missing/erroneous/duplicate csv entries, a WPT file with malformed lines, mismatches between `.csv` and `_con.csv` files, misspelled CSV entries or file names, etc, etc, etc. By testing before submitting the pull request, problems could be caught by those making the changes rather than having to wait for the next "official" site update to find out. ### Software Requirements @@ -6,7 +6,7 @@ All you will need is an ssh client to connect to the FreeBSD server (currently n ### Obtaining an account and logging in -Request an account by email to travmap@teresco.org. We will select a username and you can set a password when you first log in. To connect with PuTTY from Windows, you will create a new connection and enter "noreaster.teresco.org" in the host name, and the port number that you will be given with your account information (we run sshd on a nonstandard port to enhance security). You will be prompted for your username and password. From a Mac Terminal or other Unix-like command-line environment, you will connect with +Request an account by email to travmap@teresco.org. We will select a username and you can set a password when you first log in. To connect with PuTTY from Windows, run the "PuTTY" program (not pscp or WinSCP). You will create a new connection and enter `noreaster.teresco.org` in the host name, and the port number that you will be given with your account information (we run `sshd` on a nonstandard port to enhance security, so please don't publicize it). You will be prompted for your username and password. From a Mac Terminal or other Unix-like command-line environment, you will connect with ``` ssh -l username -p portnum noreaster.teresco.org @@ -24,7 +24,7 @@ Once you are logged in, you will see a prompt something like this: [jcool@noreaster ~]$ ``` -That is your Unix command prompt, and by default it is in a "shell" program called Bash. Basically, a shell is a way for you to issue commands directly to the operating system. Our first commands will clone the needed repositories from GitHub. Again, this assumes a GitHub username of "jcool". Type each of these, in turn, at that $ prompt. You may be prompted for a GitHub username and password. Any output from successful or unsuccessful commands will appear in your terminal, followed by a new $ prompt. +That is your Unix command prompt, and by default it is in a "shell" program called Bash. Basically, a shell is a way for you to issue commands directly to the operating system. Our first commands will clone the needed repositories from GitHub. Again, this assumes a GitHub username of "jcool". Type each of these, in turn, at that `$` prompt. You may be prompted for a GitHub username and password. Any output from successful or unsuccessful commands will appear in your terminal, followed by a new `$` prompt. ``` git clone https://github.com/jcool/HighwayData.git @@ -32,36 +32,22 @@ git clone https://github.com/TravelMapping/UserData.git git clone https://github.com/TravelMapping/DataProcessing.git ``` -Don't forget to replace "jcool" with your GitHub username into which you have forked the HighwayData repository. +Don't forget to replace "jcool" with your GitHub username into which you have forked the `HighwayData` repository. If all were successful, you should now have copies of each of the repositories in your account on the server. -### Running the site update code +### Running the site update code in "highway data check mode" -At this time, your best bet is to run the full site update program, even though you don't need everything it does. https://github.com/TravelMapping/DataProcessing/issues/73 has been created as a reminder to simplify the process for this purpose. +At this time, will run the site update program in "highway data check mode". Since for checking highway data updates, it is not necessary to complete a few parts of the process nor is it necessary to generate the large SQL file that would populate the database, the program should be run with the `-e` flag. A script called `datacheck.sh` has been provided that will run this program with appropriate parameters, and it will make sure your `HighwayData` and `UserData` repositories are up to date as well. -To do this, you will enter the following at your $ prompt: +To run it, you will enter the following at your $ prompt: ``` cd ~/DataProcessing/siteupdate/python-teresco -python3 siteupdate.py -k -``` - -The process that launches will likely run for several minutes. The "-k" omits graph generation, which is one of the slowest parts of the process. You can leave it out if you want to generate graphs. - -If the program runs to completion without reporting errors, you are likely in good shape to make your pull request with your highway data changes. If not, you have things to fix. - -### Updating before subsequent runs - -When you make changes to your data on GitHub, you will need to update your clone on noreaster to match before you run the site update process. It's also a good idea to make sure the user data and data processing clones are up-to-date as well. To do this, you will issue these commands: - -``` -cd ~/DataProcessing -git pull -cd ~/UserData -git pull -cd ~/HighwayData git pull +sh datacheck.sh ``` -At this point, you can run the site update program as described above. +The `git pull` makes sure you have the latest version of the site update program from the `DataProcessing` repository, and `sh datacheck.sh` does the real work. + +The process that launches will likely run for several minutes. If the program runs to completion without reporting errors, you are likely in good shape to make your pull request with your highway data changes. If not, you have things to fix. From 2b0074b96de8b3ee543eef4f8bc631f5d92b76be Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Mon, 21 May 2018 12:06:05 -0400 Subject: [PATCH 09/13] Fix indentation mistakes related to #73 implementation. --- siteupdate/python-teresco/siteupdate.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 7a10d77d..07ccb912 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -3233,7 +3233,7 @@ def run(self): if not first: sqlfile.write(",") first = False - sqlfile.write("('" + c[0] + "','" + c[1] + "')\n") + sqlfile.write("('" + c[0] + "','" + c[1] + "')\n") sqlfile.write(";\n") sqlfile.write('CREATE TABLE countries (code VARCHAR(3), name VARCHAR(32), PRIMARY KEY(code));\n') @@ -3243,7 +3243,7 @@ def run(self): if not first: sqlfile.write(",") first = False - sqlfile.write("('" + c[0] + "','" + c[1].replace("'","''") + "')\n") + sqlfile.write("('" + c[0] + "','" + c[1].replace("'","''") + "')\n") sqlfile.write(";\n") sqlfile.write('CREATE TABLE regions (code VARCHAR(8), name VARCHAR(48), country VARCHAR(3), continent VARCHAR(3), regiontype VARCHAR(32), PRIMARY KEY(code), FOREIGN KEY (country) REFERENCES countries(code), FOREIGN KEY (continent) REFERENCES continents(code));\n') @@ -3348,8 +3348,8 @@ def run(self): sqlfile.write("(" + s.csv_line(segment_num) + ")\n") for t in s.clinched_by: clinched_list.append("'" + str(segment_num) + "','" + t.traveler_name + "'") - segment_num += 1 - sqlfile.write(";\n") + segment_num += 1 + sqlfile.write(";\n") # maybe a separate traveler table will make sense but for now, I'll just use # the name from the .list name From 68555d9ed433bcfde097f9a112d562040b09f5e6 Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Mon, 21 May 2018 18:49:01 -0400 Subject: [PATCH 10/13] Addressed many issues raised in #73 --- siteupdate/python-teresco/siteupdate.py | 115 +++++++++++++++--------- 1 file changed, 72 insertions(+), 43 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 07ccb912..c6cbf5e6 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1996,11 +1996,12 @@ def __init__(self,filename,descr,vertices,edges,format,category): # check for duplicate root entries among Route and ConnectedRoute # data in all highway systems -print(et.et() + "Checking for duplicate list names in routes, roots in routes and connected routes.", flush=True) +print(et.et() + "Checking for duplicate list names in routes, roots in routes and connected routes.",end="",flush=True) roots = [] list_names = [] duplicate_list_names = set() for h in highway_systems: + print(".", end="",flush=True) for r in h.route_list: if r.root in roots: el.add_error("Duplicate root in route lists: " + r.root) @@ -2014,12 +2015,15 @@ def __init__(self,filename,descr,vertices,edges,format,category): con_roots = [] for h in highway_systems: + print(".", end="",flush=True) for r in h.con_route_list: for cr in r.roots: if cr.root in con_roots: el.add_error("Duplicate root in con_route lists: " + cr.root) else: con_roots.append(cr.root) +print("!", flush=True) + # Make sure every route was listed as a part of some connected route if len(roots) == len(con_roots): print("Check passed: same number of routes as connected route roots. " + str(len(roots))) @@ -2227,7 +2231,7 @@ def run(self): print() # data check: visit each system and route and check for various problems -print(et.et() + "Performing data checks.", flush=True) +print(et.et() + "Performing data checks.",end="",flush=True) # first, read in the false positives list with open(args.highwaydatapath+"/datacheckfps.csv", "rt",encoding='utf-8') as file: lines = file.readlines() @@ -2256,6 +2260,7 @@ def run(self): # perform most datachecks here (list initialized above) for h in highway_systems: + print(".",end="",flush=True) for r in h.route_list: # set to be used per-route to find label duplicates all_route_labels = set() @@ -2439,14 +2444,20 @@ def run(self): labels.append(r.point_list[i+1].label) datacheckerrors.append(DatacheckEntry(r,labels,'SHARP_ANGLE', "{0:.2f}".format(angle))) +print("!", flush=True) +print(et.et() + "Found " + str(len(datacheckerrors)) + " datacheck errors.") # now mark false positives -print(et.et() + "Marking datacheck false positives.", flush=True) +print(et.et() + "Marking datacheck false positives.",end="",flush=True) fpfile = open(args.logfilepath+'/nearmatchfps.log','w',encoding='utf-8') fpfile.write("Log file created at: " + str(datetime.datetime.now()) + "\n") toremove = [] +counter = 0 for d in datacheckerrors: #print("Checking: " + str(d)) + counter += 1 + if counter % 1000 == 0: + print(".", end="",flush=True) for fp in datacheckfps: #print("Comparing: " + str(d) + " to " + str(fp)) if d.match(fp): @@ -2462,10 +2473,14 @@ def run(self): fpfile.close() # now remove the ones we matched from the list for fp in toremove: + counter += 1 + if counter % 1000 == 0: + print(".", end="",flush=True) if fp in datacheckfps: datacheckfps.remove(fp) else: print("Matched FP entry not in list!: " + str(fp)) +print("!", flush=True) # write log of unmatched false positives from the datacheckfps.csv print(et.et() + "Writing log of unmatched datacheck FP entries.") @@ -2608,7 +2623,7 @@ def run(self): # compute lots of stats, first total mileage by route, system, overall, where # system and overall are stored in dictionaries by region -print(et.et() + "Computing stats.") +print(et.et() + "Computing stats.",end="",flush=True) # now also keeping separate totals for active only, active+preview, # and all for overall (not needed for system, as a system falls into just # one of these categories) @@ -2616,6 +2631,7 @@ def run(self): active_preview_mileage_by_region = dict() overall_mileage_by_region = dict() for h in highway_systems: + print(".",end="",flush=True) for r in h.route_list: for s in r.segment_list: segment_length = s.length() @@ -2711,8 +2727,9 @@ def run(self): segment_length/system_concurrency_count else: t_system_dict[r.region] = segment_length/system_concurrency_count +print("!", flush=True) -print(et.et() + "Writing highway data stats log file (highwaydatastats.log).") +print(et.et() + "Writing highway data stats log file (highwaydatastats.log).",flush=True) hdstatsfile = open(args.logfilepath+"/highwaydatastats.log","wt",encoding='UTF-8') hdstatsfile.write("Travel Mapping highway mileage as of " + str(datetime.datetime.now()) + '\n') active_only_miles = math.fsum(list(active_only_mileage_by_region.values())) @@ -2778,7 +2795,7 @@ def run(self): ccr_values = [] cr_values = [] # now add user clinched stats to their log entries -print(et.et() + "Creating per-traveler stats log entries and augmenting data structure.") +print(et.et() + "Creating per-traveler stats log entries and augmenting data structure.",flush=True) for t in traveler_lists: t.log_entries.append("Clinched Highway Statistics") t_active_only_miles = math.fsum(list(t.active_only_mileage_by_region.values())) @@ -2922,12 +2939,12 @@ def run(self): " preview systems") # write log files for traveler lists -print(et.et() + "Writing traveler list logs.") +print(et.et() + "Writing traveler list logs.",flush=True) for t in traveler_lists: t.write_log(args.logfilepath) # write stats csv files -print(et.et() + "Writing stats csv files.") +print(et.et() + "Writing stats csv files.",flush=True) # first, overall per traveler by region, both active only and active+preview allfile = open(args.csvstatfilepath + "/allbyregionactiveonly.csv","w",encoding='UTF-8') allfile.write("Traveler") @@ -3184,13 +3201,20 @@ def run(self): logfile = open(args.logfilepath + '/datacheck.log', 'w') logfile.write("Log file created at: " + str(datetime.datetime.now()) + "\n") logfile.write("Datacheck errors that have been flagged as false positives are not included.\n") +logfile.write("These entries should be in a format ready to paste into datacheckfps.csv.\n") if len(datacheckerrors) > 0: for d in datacheckerrors: if not d.fp: - logfile.write(str(d.route.root)) - for l in d.labels: - logfile.write(" " + l) - logfile.write(" "+d.code+" "+d.info+"\n") + logfile.write(str(d.route.root)+";") + if len(d.labels) == 0: + logfile.write(";;;;") + elif len(d.labels) == 1: + logfile.write(d.labels[0]+";;;") + elif len(d.labels) == 2: + logfile.write(d.labels[0]+";"+d.labels[1]+";;") + else: + logfile.write(d.labels[0]+";"+d.labels[1]+";"+d.labels[2]+";") + logfile.write(d.code+";"+d.info+"\n") else: logfile.write("No datacheck errors found.") logfile.close() @@ -3543,36 +3567,41 @@ def run(self): str(points) + " points and " + str(segments) + " segments.") if points != all_waypoints.size(): print("MISMATCH: all_waypoints contains " + str(all_waypoints.size()) + " waypoints!") -# compute colocation of waypoints stats -print(et.et() + "Computing waypoint colocation stats, reporting all with 8 or more colocations:") -colocate_counts = [0]*50 -largest_colocate_count = 1 -big_colocate_locations = dict() -for w in all_waypoints.point_list(): - c = w.num_colocated() - if c >= 8: - point = (w.lat, w.lng) - entry = w.route.root + " " + w.label - if point in big_colocate_locations: - the_list = big_colocate_locations[point] - the_list.append(entry) - big_colocate_locations[point] = the_list - else: - the_list = [] - the_list.append(entry) - big_colocate_locations[point] = the_list - #print(str(w) + " with " + str(c) + " other points.") - colocate_counts[c] += 1 - if c > largest_colocate_count: - largest_colocate_count = c -for place in big_colocate_locations: - the_list = big_colocate_locations[place] - print(str(place) + " is occupied by " + str(len(the_list)) + " waypoints: " + str(the_list)) -print("Waypoint colocation counts:") -unique_locations = 0 -for c in range(1,largest_colocate_count+1): - unique_locations += colocate_counts[c]//c - print("{0:6d} are each occupied by {1:2d} waypoints.".format(colocate_counts[c]//c, c)) -print("Unique locations: " + str(unique_locations)) + +if not args.errorcheck: + # compute colocation of waypoints stats + print(et.et() + "Computing waypoint colocation stats, reporting all with 8 or more colocations:") + colocate_counts = [0]*50 + largest_colocate_count = 1 + big_colocate_locations = dict() + for w in all_waypoints.point_list(): + c = w.num_colocated() + if c >= 8: + point = (w.lat, w.lng) + entry = w.route.root + " " + w.label + if point in big_colocate_locations: + the_list = big_colocate_locations[point] + the_list.append(entry) + big_colocate_locations[point] = the_list + else: + the_list = [] + the_list.append(entry) + big_colocate_locations[point] = the_list + #print(str(w) + " with " + str(c) + " other points.") + colocate_counts[c] += 1 + if c > largest_colocate_count: + largest_colocate_count = c + for place in big_colocate_locations: + the_list = big_colocate_locations[place] + print(str(place) + " is occupied by " + str(len(the_list)) + " waypoints: " + str(the_list)) + print("Waypoint colocation counts:") + unique_locations = 0 + for c in range(1,largest_colocate_count+1): + unique_locations += colocate_counts[c]//c + print("{0:6d} are each occupied by {1:2d} waypoints.".format(colocate_counts[c]//c, c)) + print("Unique locations: " + str(unique_locations)) + +if args.errorcheck: + print("Data check successful!") print("Total run time: " + et.et()) From 8f0f51b5277c52d192c3a49dea4bcf24cfa4a959 Mon Sep 17 00:00:00 2001 From: jteresco Date: Mon, 21 May 2018 18:50:52 -0400 Subject: [PATCH 11/13] Minor fixes to site update. --- siteupdate/python-teresco/localupdate.sh | 1 - siteupdate/python-teresco/siteupdate.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/siteupdate/python-teresco/localupdate.sh b/siteupdate/python-teresco/localupdate.sh index 963e6b15..1fd99446 100644 --- a/siteupdate/python-teresco/localupdate.sh +++ b/siteupdate/python-teresco/localupdate.sh @@ -18,7 +18,6 @@ graphflag= date # process command line args for arg in "$@"; do - echo "$arg" if [ "$arg" == "--nographs" ]; then # -k to siteupdate.py supresses graph generation graphflag="-k" diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 2e43c7a5..2874d808 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2163,7 +2163,7 @@ def run(self): nmpfpfile.close() nmpfile = open(args.logfilepath+'/nearmisspoints.log','w') -nmpnmp = open(args.graphfilepath+'/tm-master.nmp','w') +nmpnmp = open(args.logfilepath+'/tm-master.nmp','w') for w in all_waypoints.point_list(): if w.near_miss_points is not None: nmpline = str(w) + " NMP " From e2e47bf88ba82f1f948a59acae77aded6b16141e Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Tue, 22 May 2018 20:07:03 -0400 Subject: [PATCH 12/13] Fixes #80 --- siteupdate/python-teresco/siteupdate.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index ac447476..4f618ff8 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2797,6 +2797,7 @@ def run(self): # now add user clinched stats to their log entries print(et.et() + "Creating per-traveler stats log entries and augmenting data structure.",flush=True) for t in traveler_lists: + print(".",end="",flush=True) t.log_entries.append("Clinched Highway Statistics") t_active_only_miles = math.fsum(list(t.active_only_mileage_by_region.values())) t.log_entries.append("Overall in active systems: " + format_clinched_mi(t_active_only_miles,active_only_miles)) @@ -2937,6 +2938,7 @@ def run(self): ", Clinched " + str(t.preview_systems_clinched) + " of " + str(preview_systems) + " ({0:.1f}%)".format(100*t.preview_systems_clinched/preview_systems) + " preview systems") +print("!", flush=True) # write log files for traveler lists print(et.et() + "Writing traveler list logs.",flush=True) From 23a22244e34f678479fe02cfd753428969e54f79 Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Tue, 22 May 2018 20:29:45 -0400 Subject: [PATCH 13/13] One more minor fix for #80. --- siteupdate/python-teresco/siteupdate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 4f618ff8..d4569a02 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2795,7 +2795,7 @@ def run(self): ccr_values = [] cr_values = [] # now add user clinched stats to their log entries -print(et.et() + "Creating per-traveler stats log entries and augmenting data structure.",flush=True) +print(et.et() + "Creating per-traveler stats log entries and augmenting data structure.",end="",flush=True) for t in traveler_lists: print(".",end="",flush=True) t.log_entries.append("Clinched Highway Statistics")