From 370b00485bcea1fea88b68eea07c22fb20ac03f2 Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Fri, 26 Jan 2018 21:04:32 -0500 Subject: [PATCH 01/17] Expand Banner field to 6 characters. --- siteupdate/python-teresco/siteupdate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 122f892a..ed084bea 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -666,7 +666,7 @@ class Route: Route: the route name as would be specified in user lists Banner: the (optional) banner on the route such as 'Alt', - 'Bus', or 'Trk'. + 'Bus', or 'Trk'. Now allowed up to 6 characters Abbrev: (optional) for bannered routes or routes in multiple sections, the 3-letter abbrevation for the city or other place @@ -3243,7 +3243,7 @@ def run(self): sqlfile.write(";\n") # next, a table of highways, with the same fields as in the first line -sqlfile.write('CREATE TABLE routes (systemName VARCHAR(10), region VARCHAR(8), route VARCHAR(16), banner VARCHAR(3), abbrev VARCHAR(3), city VARCHAR(100), root VARCHAR(32), mileage FLOAT, PRIMARY KEY(root), FOREIGN KEY (systemName) REFERENCES systems(systemName));\n') +sqlfile.write('CREATE TABLE routes (systemName VARCHAR(10), region VARCHAR(8), route VARCHAR(16), banner VARCHAR(6), abbrev VARCHAR(3), city VARCHAR(100), root VARCHAR(32), mileage FLOAT, PRIMARY KEY(root), FOREIGN KEY (systemName) REFERENCES systems(systemName));\n') sqlfile.write('INSERT INTO routes VALUES\n') first = True for h in highway_systems: @@ -3255,7 +3255,7 @@ def run(self): sqlfile.write(";\n") # connected routes table, but only first "root" in each in this table -sqlfile.write('CREATE TABLE connectedRoutes (systemName VARCHAR(10), route VARCHAR(16), banner VARCHAR(3), groupName VARCHAR(100), firstRoot VARCHAR(32), mileage FLOAT, PRIMARY KEY(firstRoot), FOREIGN KEY (firstRoot) REFERENCES routes(root));\n') +sqlfile.write('CREATE TABLE connectedRoutes (systemName VARCHAR(10), route VARCHAR(16), banner VARCHAR(6), groupName VARCHAR(100), firstRoot VARCHAR(32), mileage FLOAT, PRIMARY KEY(firstRoot), FOREIGN KEY (firstRoot) REFERENCES routes(root));\n') sqlfile.write('INSERT INTO connectedRoutes VALUES\n') first = True for h in highway_systems: From b719e6d00a111c52c46fc25cbbd093456c5acc33 Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Mon, 29 Jan 2018 22:35:03 -0500 Subject: [PATCH 02/17] First complete hashing of route names attempt. --- siteupdate/python-teresco/siteupdate.py | 127 ++++++++++++------------ 1 file changed, 64 insertions(+), 63 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index ed084bea..93701a00 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -944,7 +944,7 @@ class TravelerList: start_waypoint end_waypoint """ - def __init__(self,travelername,systems,path="../../../UserData/list_files"): + def __init__(self,travelername,systems,route_hash,path="../../../UserData/list_files"): self.list_entries = [] self.clinched_segments = set() self.traveler_name = travelername[:-5] @@ -965,72 +965,65 @@ def __init__(self,travelername,systems,path="../../../UserData/list_files"): continue # find the root that matches in some system and when we do, match labels - lineDone = False - for h in systems: - for r in h.route_list: - if r.region.lower() != fields[0].lower(): - continue - route_entry = fields[1].lower() - route_match = False - for a in r.alt_route_names: - if route_entry == a.lower(): - self.log_entries.append("Note: replacing deprecated route name " + fields[1] + " with canonical name " + r.list_entry_name() + " in line " + line) - route_match = True - break - if route_match or (r.list_entry_name().lower() == route_entry): - lineDone = True # we'll either have success or failure here - if h.devel(): - self.log_entries.append("Ignoring line matching highway in system in development: " + line) - break - #print("Route match with " + str(r)) - # r is a route match, r.root is our root, and we need to find - # canonical waypoint labels, ignoring case and leading "+" or "*" when matching - canonical_waypoints = [] - canonical_waypoint_indices = [] - checking_index = 0; - for w in r.point_list: - lower_label = w.label.lower().strip("+*") - list_label_1 = fields[2].lower().strip("*") - list_label_2 = fields[3].lower().strip("*") + route_entry = fields[1].lower() + lookup = fields[0].lower() + ' ' + route_entry + if lookup not in route_hash: + self.log_entries.append("Unknown region/highway combo in line: " + line) + else: + r = route_hash[lookup] + for a in r.alt_route_names: + if route_entry == a.lower(): + self.log_entries.append("Note: deprecated route name " + fields[1] + " -> canonical name " + r.list_entry_name() + " in line " + line) + break + + if r.system.devel(): + self.log_entries.append("Ignoring line matching highway in system in development: " + line) + break + # r is a route match, r.root is our root, and we need to find + # canonical waypoint labels, ignoring case and leading + # "+" or "*" when matching + canonical_waypoints = [] + canonical_waypoint_indices = [] + checking_index = 0; + for w in r.point_list: + lower_label = w.label.lower().strip("+*") + list_label_1 = fields[2].lower().strip("*") + list_label_2 = fields[3].lower().strip("*") + if list_label_1 == lower_label or list_label_2 == lower_label: + canonical_waypoints.append(w) + canonical_waypoint_indices.append(checking_index) + r.labels_in_use.add(lower_label.upper()) + else: + for alt in w.alt_labels: + lower_label = alt.lower().strip("+") if list_label_1 == lower_label or list_label_2 == lower_label: canonical_waypoints.append(w) canonical_waypoint_indices.append(checking_index) r.labels_in_use.add(lower_label.upper()) - else: - for alt in w.alt_labels: - lower_label = alt.lower().strip("+") - if list_label_1 == lower_label or list_label_2 == lower_label: - canonical_waypoints.append(w) - canonical_waypoint_indices.append(checking_index) - r.labels_in_use.add(lower_label.upper()) - # if we have not yet used this alt label, remove it from the unused list - if lower_label.upper() in r.unused_alt_labels: - r.unused_alt_labels.remove(lower_label.upper()) + # if we have not yet used this alt label, remove it from the unused list + if lower_label.upper() in r.unused_alt_labels: + r.unused_alt_labels.remove(lower_label.upper()) - checking_index += 1 - if len(canonical_waypoints) != 2: - self.log_entries.append("Waypoint label(s) not found in line: " + line) - else: - self.list_entries.append(ClinchedSegmentEntry(line, r.root, \ - canonical_waypoints[0].label, \ - canonical_waypoints[1].label)) - # find the segments we just matched and store this traveler with the - # segments and the segments with the traveler (might not need both - # ultimately) - #start = r.point_list.index(canonical_waypoints[0]) - #end = r.point_list.index(canonical_waypoints[1]) - start = canonical_waypoint_indices[0] - end = canonical_waypoint_indices[1] - for wp_pos in range(start,end): - hs = r.segment_list[wp_pos] #r.get_segment(r.point_list[wp_pos], r.point_list[wp_pos+1]) - hs.add_clinched_by(self) - if hs not in self.clinched_segments: - self.clinched_segments.add(hs) - - if lineDone: - break - if not lineDone: - self.log_entries.append("Unknown region/highway combo in line: " + line) + checking_index += 1 + if len(canonical_waypoints) != 2: + self.log_entries.append("Waypoint label(s) not found in line: " + line) + else: + self.list_entries.append(ClinchedSegmentEntry(line, r.root, \ + canonical_waypoints[0].label, \ + canonical_waypoints[1].label)) + # find the segments we just matched and store this traveler with the + # segments and the segments with the traveler (might not need both + # ultimately) + #start = r.point_list.index(canonical_waypoints[0]) + #end = r.point_list.index(canonical_waypoints[1]) + start = canonical_waypoint_indices[0] + end = canonical_waypoint_indices[1] + for wp_pos in range(start,end): + hs = r.segment_list[wp_pos] #r.get_segment(r.point_list[wp_pos], r.point_list[wp_pos+1]) + hs.add_clinched_by(self) + if hs not in self.clinched_segments: + self.clinched_segments.add(hs) + self.log_entries.append("Processed " + str(len(self.list_entries)) + \ " good lines marking " +str(len(self.clinched_segments)) + \ " segments traveled.") @@ -2473,6 +2466,14 @@ def run(self): fpfile.write("No unmatched FP entries.") fpfile.close() +# Create hash table for faster lookup of routes by list file name +print(et.et() + "Creating route hash table for list processing:",flush=True) +route_hash = dict() +for h in highway_systems: + for r in h.route_list: + route_hash[(r.region + ' ' + r.list_entry_name()).lower()] = r + for a in r.alt_route_names: + route_hash[(r.region + ' ' + a).lower()] = r # Create a list of TravelerList objects, one per person traveler_lists = [] @@ -2481,7 +2482,7 @@ def run(self): for t in traveler_ids: if t.endswith('.list'): print(" " + t,end="",flush=True) - traveler_lists.append(TravelerList(t,highway_systems,args.userlistfilepath)) + traveler_lists.append(TravelerList(t,highway_systems,route_hash,args.userlistfilepath)) print(" processed " + str(len(traveler_lists)) + " traveler list files.") # Read updates.csv file, just keep in the fields array for now since we're From 8df0f33eeb59902e8e895130b5413a89299793ce Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Tue, 30 Jan 2018 15:55:28 -0500 Subject: [PATCH 03/17] Fix new hashtable-based list entry lookup. --- siteupdate/python-teresco/siteupdate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 93701a00..6e857f0d 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -978,7 +978,7 @@ def __init__(self,travelername,systems,route_hash,path="../../../UserData/list_f if r.system.devel(): self.log_entries.append("Ignoring line matching highway in system in development: " + line) - break + continue # r is a route match, r.root is our root, and we need to find # canonical waypoint labels, ignoring case and leading # "+" or "*" when matching From 964ff4b9dda42f869b129df238aa8c2005103d6b Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Tue, 30 Jan 2018 21:51:43 -0500 Subject: [PATCH 04/17] Added noxfer option to site update control script. --- siteupdate/python-teresco/siteupdate.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/siteupdate/python-teresco/siteupdate.sh b/siteupdate/python-teresco/siteupdate.sh index 36d539c6..06db3131 100644 --- a/siteupdate/python-teresco/siteupdate.sh +++ b/siteupdate/python-teresco/siteupdate.sh @@ -2,6 +2,7 @@ # set -e read_data=1 +transfer=1 logdir=logs statdir=stats graphdir=graphs @@ -16,6 +17,9 @@ if [ $# -eq 1 ]; then if [ "$1" == "--nographs" ]; then graphflag="-k" fi + if [ "$1" == "--noxfer" ]; then + transfer=0 + fi fi if [ "$read_data" == "1" ]; then echo "siteupdate.sh: launching siteupdate.py" @@ -28,6 +32,10 @@ else echo "siteupdate.sh: SKIPPING siteupdate.py" fi date +if [ "$transfer" == "0" ]; then + echo "siteupdate.sh: SKIPPING file transfers and DB update" + exit 0 +fi echo "siteupdate.sh: Bzipping TravelMapping.sql file" bzip2 -9f TravelMapping.sql echo "siteupdate.sh: Transferring TravelMapping.sql.bz2 to blizzard" From 64eaff1bfe78fc96e574917fa51d35ae289f04ac Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Mon, 5 Mar 2018 15:02:03 -0500 Subject: [PATCH 05/17] Initial version --- SETUP.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 SETUP.md diff --git a/SETUP.md b/SETUP.md new file mode 100644 index 00000000..8c91d383 --- /dev/null +++ b/SETUP.md @@ -0,0 +1,2 @@ +This document is intended to describe how one can set up to run TravelMapping's site update program to generate the same logs, stats, and database file that are produced as part of TM's regular site update process. + From 053bed3803ebe31cedf4b61597c7263b5f1e9903 Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Mon, 5 Mar 2018 15:17:04 -0500 Subject: [PATCH 06/17] First few steps of instructions --- SETUP.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/SETUP.md b/SETUP.md index 8c91d383..93254180 100644 --- a/SETUP.md +++ b/SETUP.md @@ -1,2 +1,13 @@ -This document is intended to describe how one can set up to run TravelMapping's site update program to generate the same logs, stats, and database file that are produced as part of TM's regular site update process. +This document is intended to describe how one can set up an environment to run TravelMapping's site update program to generate the same logs, stats, and database file that are produced as part of TM's regular site update process. A separate document is planned in the [Web repository](https://github.com/TravelMapping/Web/) to describe how to take the information generated by the update to populate a database and configure the Web-facing code to use it. + +### Cloning Needed Repositories + +Information from three repositories is needed to run the site update process: + +1. [DataProcessing](https://github.com/TravelMapping/DataProcessing/) +2. [HighwayData](https://github.com/TravelMapping/HighwayData/) +3. [UserData](https://github.com/TravelMapping/UserData/) + +These should be cloned into the same parent directory. Typically, this might be called "travelmapping" and sit off the user's home directory. Below, we will assume that directory is in an environment variable TMBASE, so we refer to it as $(TMBASE). + From b3b20927bced00db650730768cd51b3b8d2db4c4 Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Mon, 5 Mar 2018 16:16:33 -0500 Subject: [PATCH 07/17] Note about needing a Unix-like environment --- SETUP.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SETUP.md b/SETUP.md index 93254180..3d083ef5 100644 --- a/SETUP.md +++ b/SETUP.md @@ -1,4 +1,4 @@ -This document is intended to describe how one can set up an environment to run TravelMapping's site update program to generate the same logs, stats, and database file that are produced as part of TM's regular site update process. A separate document is planned in the [Web repository](https://github.com/TravelMapping/Web/) to describe how to take the information generated by the update to populate a database and configure the Web-facing code to use it. +This document is intended to describe how one can set up an environment to run TravelMapping's site update program to generate the same logs, stats, and database file that are produced as part of TM's regular site update process. A separate document is planned in the [Web repository](https://github.com/TravelMapping/Web/) to describe how to take the information generated by the update to populate a database and configure the Web-facing code to use it. The expectation is that this is run in a Unix-like environment such as the Mac OS X Terminal, or Linux or FreeBSD. It is likely the same or similar commands will work on Windows in something like Cygwin or Git Bash, but this has not been tested. ### Cloning Needed Repositories From a48aa24acc09fc80050bac396ae4837bb7b3c780 Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Mon, 5 Mar 2018 20:21:05 -0500 Subject: [PATCH 08/17] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 805b2aef..b51da840 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ # DataProcessing -Data Processing Scripts and Programs for Travel Mapping Project +Data Processing Scripts and Programs for the Travel Mapping Project. From 64f5c157a3460bd6e89170e2082043fe477308bd Mon Sep 17 00:00:00 2001 From: jteresco Date: Fri, 11 May 2018 10:35:24 -0400 Subject: [PATCH 09/17] Added gitignore for produced directories. --- siteupdate/python-teresco/.gitignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 siteupdate/python-teresco/.gitignore diff --git a/siteupdate/python-teresco/.gitignore b/siteupdate/python-teresco/.gitignore new file mode 100644 index 00000000..51ac628b --- /dev/null +++ b/siteupdate/python-teresco/.gitignore @@ -0,0 +1,3 @@ +graphs +logs +nmp_merged From aecfe457563f9dd1da3636a99afd3ade96d14177 Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Fri, 11 May 2018 11:45:46 -0400 Subject: [PATCH 10/17] A few more items. --- SETUP.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/SETUP.md b/SETUP.md index 3d083ef5..8e1f37b2 100644 --- a/SETUP.md +++ b/SETUP.md @@ -1,5 +1,11 @@ This document is intended to describe how one can set up an environment to run TravelMapping's site update program to generate the same logs, stats, and database file that are produced as part of TM's regular site update process. A separate document is planned in the [Web repository](https://github.com/TravelMapping/Web/) to describe how to take the information generated by the update to populate a database and configure the Web-facing code to use it. The expectation is that this is run in a Unix-like environment such as the Mac OS X Terminal, or Linux or FreeBSD. It is likely the same or similar commands will work on Windows in something like Cygwin or Git Bash, but this has not been tested. +### Needed Software + +Standard tools expected include bash, bzip2, ssh. + +The system should have a Python3 installation. As of this writing, Python 3.6.4 is being used. Below, we will assume that Python can be launched with the command "python3". + ### Cloning Needed Repositories Information from three repositories is needed to run the site update process: @@ -10,4 +16,9 @@ Information from three repositories is needed to run the site update process: These should be cloned into the same parent directory. Typically, this might be called "travelmapping" and sit off the user's home directory. Below, we will assume that directory is in an environment variable TMBASE, so we refer to it as $(TMBASE). +Once all repositories have been updated to match the latest versions on GitHub (using "git pull"), the site update program can be run. To run the basic site update: +``` +cd $(TMBASE)/DataProcessing/siteupdate/python-teresco +python3 siteupdate.py +``` From fc35a7aacfbd53b8526d6ca0891f0e5940ae27ef Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Fri, 11 May 2018 11:48:04 -0400 Subject: [PATCH 11/17] Mention siteupdate.sh --- SETUP.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/SETUP.md b/SETUP.md index 8e1f37b2..54a72e8a 100644 --- a/SETUP.md +++ b/SETUP.md @@ -16,9 +16,17 @@ Information from three repositories is needed to run the site update process: These should be cloned into the same parent directory. Typically, this might be called "travelmapping" and sit off the user's home directory. Below, we will assume that directory is in an environment variable TMBASE, so we refer to it as $(TMBASE). -Once all repositories have been updated to match the latest versions on GitHub (using "git pull"), the site update program can be run. To run the basic site update: +Once all repositories have been updated to match the latest versions on GitHub (using "git pull"), the site update program can be run. To run the basic data processing to ensure that all data can be loaded correcly and that no other errors are encountered: ``` cd $(TMBASE)/DataProcessing/siteupdate/python-teresco python3 siteupdate.py ``` + +There is also a collection of bash scripts that run this program, typically launched with + +``` +sh siteupdate.sh +``` + +that also transfers files to the DB and web servers and reloads the DB. From 78dadb33e73258364bd4a553e53514070022d094 Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Sat, 12 May 2018 21:15:28 -0400 Subject: [PATCH 12/17] Update SETUP.md --- SETUP.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SETUP.md b/SETUP.md index 54a72e8a..dabeb726 100644 --- a/SETUP.md +++ b/SETUP.md @@ -1,4 +1,4 @@ -This document is intended to describe how one can set up an environment to run TravelMapping's site update program to generate the same logs, stats, and database file that are produced as part of TM's regular site update process. A separate document is planned in the [Web repository](https://github.com/TravelMapping/Web/) to describe how to take the information generated by the update to populate a database and configure the Web-facing code to use it. The expectation is that this is run in a Unix-like environment such as the Mac OS X Terminal, or Linux or FreeBSD. It is likely the same or similar commands will work on Windows in something like Cygwin or Git Bash, but this has not been tested. +This document is intended to describe how one can set up an environment to run TravelMapping's site update program to generate the same logs, stats, and database file that are produced as part of TM's regular site update process. A separate document is planned in the [Web repository](https://github.com/TravelMapping/Web/) to describe how to take the information generated by the update to populate a database and install and configure the Web-facing code to use it. The expectation is that this is run in a Unix-like environment such as the Mac OS X Terminal, or Linux or FreeBSD. It is likely the same or similar commands will work on Windows in something like Cygwin or Git Bash, but this has not been tested. A [separate document](RUNNING.md) describes how to use a shell account on TravelMapping's server to test your own changes to the highway data before submitting a pull request. ### Needed Software From a87a46e41684bf49563a17476076b5d36a0b2e2b Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Sat, 12 May 2018 21:48:09 -0400 Subject: [PATCH 13/17] Instructions for testing highway data changes --- RUNNING.md | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 RUNNING.md diff --git a/RUNNING.md b/RUNNING.md new file mode 100644 index 00000000..dd31fd64 --- /dev/null +++ b/RUNNING.md @@ -0,0 +1,65 @@ +This document describes how to use the site update code on the Travel Mapping server to test your changes to highway data before submitting a pull request. The idea is to speed the process of getting your changes and updates into the site by finding things like missing or misnamed files, missing/erroneous/duplicate csv entries, a WPT file with malformed lines, mismatches between .csv and _con.csv files, misspelled CSV entries or file names, etc, etc, etc. By testing before submitting the pull request, problems could be caught by those making the changes rather than having to wait for the next "official" site update to find out. + +### Software Requirements + +All you will need is an ssh client to connect to the FreeBSD server (currently noreaster.teresco.org) that is used by this project. If you are on Windows, the most popular option is likely [PuTTY](https://www.chiark.greenend.org.uk/~sgtatham/putty/). If you are on a Mac, your Terminal application already has what you need in the command-line ssh client. Same for other Unix-like environments (Linux, FreeBSD, etc.). + +### Obtaining an account and logging in + +Request an account by email to travmap@teresco.org. We will select a username and you can set a password when you first log in. To connect with PuTTY from Windows, you will create a new connection and enter "noreaster.teresco.org" in the host name, and the port number that you will be given with your account information (we run sshd on a nonstandard port to enhance security). You will be prompted for your username and password. From a Mac Terminal or other Unix-like command-line environment, you will connect with + +``` +ssh -l username -p portnum noreaster.teresco.org +``` + +Where you would replace "username" with your assigned username, and "portnum" with the number you are given with your account information. + +### First time setup + +The first time you connect, you will need to clone the appropriate GitHub repositories into your account. You will need the HighwayData repository, the UserData repository, and the DataProcessing repository. Most often, you will want your own fork of the first, and the master versions of the latter two, as this process is intended to help test changes to highway data. We'll assume that's the case here, and will use "jcool" as the GitHub account into which you have forked the HighwayData repository. + +Once you are logged in, you will see a prompt something like this: + +``` +[jcool@noreaster ~]$ +``` + +That is your Unix command prompt, and by default it is in a "shell" program called Bash. Basically, a shell is a way for you to issue commands directly to the operating system. Our first commands will clone the needed repositories from GitHub. Again, this assumes a GitHub username of "jcool". Type each of these, in turn, at that $ prompt. You may be prompted for a GitHub username and password. Any output from successful or unsuccessful commands will appear in your terminal, followed by a new $ prompt. + +``` +git clone https://github.com/jcool/HighwayData.git +git clone https://github.com/TravelMapping/UserData.git +git clone https://github.com/TravelMapping/DataProcessing.git +``` + +If all were successful, you should now have copies of each of the repositories in your account on the server. + +### Running the site update code + +At this time, your best bet is to run the full site update program, even though you don't need everything it does. https://github.com/TravelMapping/DataProcessing/issues/73 has been created as a reminder to simplify the process for this purpose. + +To do this, you will enter the following at your $ prompt: + +``` +cd ~/DataProcessing/siteupdate/python-teresco +python3 siteupdate.py -k +``` + +The process that launches will likely run for several minutes. The "-k" omits graph generation, which is one of the slowest parts of the process. You can leave it out if you want to generate graphs. + +If the program runs to completion without reporting errors, you are likely in good shape to make your pull request with your highway data changes. If not, you have things to fix. + +### Updating before subsequent runs + +When you make changes to your data on GitHub, you will need to update your clone on noreaster to match before you run the site update process. It's also a good idea to make sure the user data and data processing clones are up-to-date as well. To do this, you will issue these commands: + +``` +cd ~/DataProcessing +git pull +cd ~/UserData +git pull +cd ~/HighwayData +git pull +``` + +At this point, you can run the site update program as described above. From b78fed89fbab6b87f091b9d34059b4125a8e49c6 Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Sat, 12 May 2018 21:49:45 -0400 Subject: [PATCH 14/17] Minor update --- RUNNING.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/RUNNING.md b/RUNNING.md index dd31fd64..52e066ab 100644 --- a/RUNNING.md +++ b/RUNNING.md @@ -32,6 +32,8 @@ git clone https://github.com/TravelMapping/UserData.git git clone https://github.com/TravelMapping/DataProcessing.git ``` +Don't forget to replace "jcool" with your GitHub username into which you have forked the HighwayData repository. + If all were successful, you should now have copies of each of the repositories in your account on the server. ### Running the site update code From 31d4576e38411f2e80043c2b9f09815acd9d5e74 Mon Sep 17 00:00:00 2001 From: jteresco Date: Tue, 15 May 2018 21:46:34 -0400 Subject: [PATCH 15/17] New local update procedure, field size fixes for DB tables. --- siteupdate/python-teresco/localupdate.sh | 84 ++++++++++++++++++++++++ siteupdate/python-teresco/siteupdate.py | 8 +-- 2 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 siteupdate/python-teresco/localupdate.sh diff --git a/siteupdate/python-teresco/localupdate.sh b/siteupdate/python-teresco/localupdate.sh new file mode 100644 index 00000000..963e6b15 --- /dev/null +++ b/siteupdate/python-teresco/localupdate.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# +# script to run a site update on the same system as the web front end +# +# Jim Teresco, Tue May 15 15:57:28 EDT 2018 +# +set -e +install=1 +pull=1 +tmbase=$HOME/travelmapping +tmwebbase=/home/www/tm +datestr=`date '+%Y-%m-%d@%H:%M:%S'` +logdir=logs +statdir=stats +graphdir=graphs +nmpmdir=nmp_merged +graphflag= +date +# process command line args +for arg in "$@"; do + echo "$arg" + if [ "$arg" == "--nographs" ]; then + # -k to siteupdate.py supresses graph generation + graphflag="-k" + fi + if [ "$arg" == "--noinstall" ]; then + install=0 + fi + if [ "$arg" == "--nopull" ]; then + pull=0 + fi + shift +done +if [ "$pull" == "1" ]; then + echo "$0: updating TM repositories" + (cd $tmbase/HighwayData; git pull) + (cd $tmbase/UserData; git pull) +fi + +echo "$0: creating directories" +mkdir -p $datestr/$logdir $datestr/$statdir $datestr/$nmpmdir +if [ "$graphflag" != "-k" ]; then + mkdir -p $datestr/$graphdir +fi + +echo "$0: launching siteupdate.py" +PYTHONIOENCODING='utf-8' ./siteupdate.py -d TravelMapping-$datestr $graphflag -l $datestr/$logdir -c $datestr/$statdir -g $datestr/$graphdir -n $datestr/$nmpmdir | tee $datestr/$logdir/siteupdate.log 2>&1 || exit 1 +date + +if [ "$install" == "0" ]; then + echo "$0: SKIPPING file copies and DB update" + exit 0 +fi +echo "$0: installing logs, stats, nmp_merged, graphs, archiving old contents in /tmp/$datestr" +mkdir -p /tmp/$datestr +mv $tmwebbase/$logdir /tmp/$datestr +mv $datestr/$logdir $tmwebbase +mv $tmwebbase/$statdir /tmp/$datestr +mv $datestr/$statdir $tmwebbase +mv $tmwebbase/$nmpmdir /tmp/$datestr +mv $datestr/$nmpmdir $tmwebbase +if [ "$graphflag" != "-k" ]; then + mv $tmwebbase/$graphdir /tmp/$datestr + mv $datestr/$graphdir $tmwebbase +fi +rmdir $datestr +echo "$0: switching to DB copy" +ln -sf $tmwebbase/lib/tm.conf.updating $tmwebbase/lib/tm.conf +touch $tmwebbase/dbupdating +echo "$0: loading primary DB" +mysql --defaults-group-suffix=tmapadmin -u travmapadmin TravelMapping < TravelMapping-$datestr.sql +/bin/rm $tmwebbase/dbupdating +echo "$0: switching to primary DB" +ln -sf $tmwebbase/lib/tm.conf.standard $tmwebbase/lib/tm.conf +echo "$0: loading DB copy" +mysql --defaults-group-suffix=tmapadmin -u travmapadmin TravelMappingCopy < TravelMapping-$datestr.sql +echo "$0: complete" +#echo "$0: sending email notification" +#mailx -s "Travel Mapping Site Update Complete" travelmapping-siteupdates@teresco.org < Date: Tue, 15 May 2018 23:11:41 -0400 Subject: [PATCH 16/17] Temporarily create graphs directory even when no graphs generated. --- siteupdate/python-teresco/localupdate.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/siteupdate/python-teresco/localupdate.sh b/siteupdate/python-teresco/localupdate.sh index 963e6b15..7eb22959 100644 --- a/siteupdate/python-teresco/localupdate.sh +++ b/siteupdate/python-teresco/localupdate.sh @@ -18,7 +18,6 @@ graphflag= date # process command line args for arg in "$@"; do - echo "$arg" if [ "$arg" == "--nographs" ]; then # -k to siteupdate.py supresses graph generation graphflag="-k" @@ -38,10 +37,11 @@ if [ "$pull" == "1" ]; then fi echo "$0: creating directories" -mkdir -p $datestr/$logdir $datestr/$statdir $datestr/$nmpmdir -if [ "$graphflag" != "-k" ]; then - mkdir -p $datestr/$graphdir -fi +mkdir -p $datestr/$logdir $datestr/$statdir $datestr/$nmpmdir $datestr/$graphdir +# put this back later: need to put tm-master.nmp somewhere else +#if [ "$graphflag" != "-k" ]; then +# mkdir -p $datestr/$graphdir +#fi echo "$0: launching siteupdate.py" PYTHONIOENCODING='utf-8' ./siteupdate.py -d TravelMapping-$datestr $graphflag -l $datestr/$logdir -c $datestr/$statdir -g $datestr/$graphdir -n $datestr/$nmpmdir | tee $datestr/$logdir/siteupdate.log 2>&1 || exit 1 From 0ff204e3b17279515f1f4154b4ad24f8e0c52229 Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Wed, 16 May 2018 14:37:52 -0400 Subject: [PATCH 17/17] Fix missing str conversion. --- siteupdate/python-teresco/siteupdate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 6e857f0d..136fd45f 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1328,7 +1328,7 @@ def __init__(self,graph,segment=None,vertex_info=None): self.vertex2.incident_collapsed_edges.remove(edge1) removed += 1 if removed != 1: - print("ERROR: edge1 " + str(edge1) + " removed from " + removed + " adjacency lists instead of 1.") + print("ERROR: edge1 " + str(edge1) + " removed from " + str(removed) + " adjacency lists instead of 1.") removed = 0 if edge2 in self.vertex1.incident_collapsed_edges: self.vertex1.incident_collapsed_edges.remove(edge2) @@ -1337,7 +1337,7 @@ def __init__(self,graph,segment=None,vertex_info=None): self.vertex2.incident_collapsed_edges.remove(edge2) removed += 1 if removed != 1: - print("ERROR: edge2 " + str(edge2) + " removed from " + removed + " adjacency lists instead of 1.") + print("ERROR: edge2 " + str(edge2) + " removed from " + str(removed) + " adjacency lists instead of 1.") self.vertex1.incident_collapsed_edges.append(self) self.vertex2.incident_collapsed_edges.append(self)