sync, 2018-07-16

siteupdate/python-teresco/datacheck.sh

@@ -32,7 +32,7 @@ if [ "$pull" == "1" ]; then
 fi
 
 echo "$0: creating directories"
-mkdir -p $logdir $statdir
+mkdir -p $logdir/users $statdir
 
 echo "$0: launching siteupdate.py"
 PYTHONIOENCODING='utf-8' ./siteupdate.py -e -l $logdir -c $statdir | tee $logdir/siteupdate.log 2>&1 || exit 1

siteupdate/python-teresco/localupdate.sh

@@ -9,10 +9,11 @@ install=1
 pull=1
 tmbase=$HOME/travelmapping
 tmwebbase=/home/www/tm
+tmpdir=/home/tmp/tm
 datestr=`date '+%Y-%m-%d@%H:%M:%S'`
 logdir=logs
 statdir=stats
-graphdir=graphs
+graphdir=graphdata
 nmpmdir=nmp_merged
 graphflag=
 date
@@ -37,11 +38,10 @@ if [ "$pull" == "1" ]; then
 fi
 
 echo "$0: creating directories"
-mkdir -p $datestr/$logdir $datestr/$statdir $datestr/$nmpmdir $datestr/$graphdir
-# put this back later: need to put tm-master.nmp somewhere else
-#if [ "$graphflag" != "-k" ]; then
-# mkdir -p $datestr/$graphdir
-#fi
+mkdir -p $datestr/$logdir/users $datestr/$statdir $datestr/$nmpmdir
+if [ "$graphflag" != "-k" ]; then
+ mkdir -p $datestr/$graphdir
+fi
 
 echo "$0: launching siteupdate.py"
 PYTHONIOENCODING='utf-8' ./siteupdate.py -d TravelMapping-$datestr $graphflag -l $datestr/$logdir -c $datestr/$statdir -g $datestr/$graphdir -n $datestr/$nmpmdir | tee $datestr/$logdir/siteupdate.log 2>&1 || exit 1
@@ -51,16 +51,16 @@ if [ "$install" == "0" ]; then
  echo "$0: SKIPPING file copies and DB update"
  exit 0
 fi
-echo "$0: installing logs, stats, nmp_merged, graphs, archiving old contents in /tmp/$datestr"
-mkdir -p /tmp/$datestr
-mv $tmwebbase/$logdir /tmp/$datestr
+echo "$0: installing logs, stats, nmp_merged, graphs, archiving old contents in $tmpdir/$datestr"
+mkdir -p $tmpdir/$datestr
+mv $tmwebbase/$logdir $tmpdir/$datestr
 mv $datestr/$logdir $tmwebbase
-mv $tmwebbase/$statdir /tmp/$datestr
+mv $tmwebbase/$statdir $tmpdir/$datestr
 mv $datestr/$statdir $tmwebbase
-mv $tmwebbase/$nmpmdir /tmp/$datestr
+mv $tmwebbase/$nmpmdir $tmpdir/$datestr
 mv $datestr/$nmpmdir $tmwebbase
 if [ "$graphflag" != "-k" ]; then
- mv $tmwebbase/$graphdir /tmp/$datestr
+ mv $tmwebbase/$graphdir $tmpdir/$datestr
  mv $datestr/$graphdir $tmwebbase
 fi
 rmdir $datestr
@@ -74,11 +74,13 @@ echo "$0: switching to primary DB"
 ln -sf $tmwebbase/lib/tm.conf.standard $tmwebbase/lib/tm.conf
 echo "$0: loading DB copy"
 mysql --defaults-group-suffix=tmapadmin -u travmapadmin TravelMappingCopy < TravelMapping-$datestr.sql
+echo "$0: moving sql file to archive"
+mv TravelMapping-$datestr.sql $tmpdir
+echo "$0: sending email notification"
+mailx -s "Travel Mapping Site Update Complete" travelmapping-siteupdates@teresco.org <<EOF
+A Travel Mapping site update has just successfully completed.
+The complete log is available at http://travelmapping.net/logs/siteupdate.log .
+Please report any problems to travmap@teresco.org .
+EOF
 echo "$0: complete"
-#echo "$0: sending email notification"
-#mailx -s "Travel Mapping Site Update Complete" travelmapping-siteupdates@teresco.org <<EOF
-#A Travel Mapping site update has just successfully completed.
-#The complete log is available at http://tm.teresco.org/logs/siteupdate.log .
-#Please report any problems to travmap@teresco.org .
-#EOF
 date

siteupdate/python-teresco/siteupdate.py

@@ -254,10 +254,6 @@ def __str__(self):
  ans = ans + " (" + str(self.lat) + "," + str(self.lng) + ")"
  return ans
 
- def sql_insert_command(self,tablename,id):
- """return sql command to insert into a table"""
- return "INSERT INTO " + tablename + " VALUES ('" + str(id) + "','" + self.label + "','" + str(self.lat) + "','" + str(self.lng) + "','" + self.route.root + "');"
-
  def csv_line(self,id):
  """return csv line to insert into a table"""
  return "'" + str(id) + "','" + self.label + "','" + str(self.lat) + "','" + str(self.lng) + "','" + self.route.root + "'"
@@ -587,10 +583,6 @@ def add_clinched_by(self,traveler):
  else:
  return False
 
- def sql_insert_command(self,tablename,id):
- """return sql command to insert into a table"""
- return "INSERT INTO " + tablename + " VALUES ('" + str(id) + "','" + str(self.waypoint1.point_num) + "','" + str(self.waypoint2.point_num) + "','" + self.route.root + "');"
-
  def csv_line(self,id):
  """return csv line to insert into a table"""
  return "'" + str(id) + "','" + str(self.waypoint1.point_num) + "','" + str(self.waypoint2.point_num) + "','" + self.route.root + "'"
@@ -704,6 +696,7 @@ def __init__(self,line,system,el):
  self.unused_alt_labels = set()
  self.segment_list = []
  self.mileage = 0.0
+ self.rootOrder = -1 # order within connected route
 
  def __str__(self):
  """printable version of the object"""
@@ -778,17 +771,11 @@ def find_segment_by_waypoints(self,w1,w2):
  return s
  return None
 
- def sql_insert_command(self,tablename):
- """return sql command to insert into a table"""
- # note: alt_route_names does not need to be in the db since
- # list preprocessing uses alt or canonical and no longer cares
- return "INSERT INTO " + tablename + " VALUES ('" + self.system.systemname + "','" + self.region + "','" + self.route + "','" + self.banner + "','" + self.abbrev + "','" + self.city + "','" + self.root + "');";
-
  def csv_line(self):
  """return csv line to insert into a table"""
  # note: alt_route_names does not need to be in the db since
  # list preprocessing uses alt or canonical and no longer cares
- return "'" + self.system.systemname + "','" + self.region + "','" + self.route + "','" + self.banner + "','" + self.abbrev + "','" + self.city + "','" + self.root + "','" + str(self.mileage) + "'";
+ return "'" + self.system.systemname + "','" + self.region + "','" + self.route + "','" + self.banner + "','" + self.abbrev + "','" + self.city + "','" + self.root + "','" + str(self.mileage) + "','" + str(self.rootOrder) + "'";
 
  def readable_name(self):
  """return a string for a human-readable route name"""
@@ -834,6 +821,7 @@ def __init__(self,line,system,el):
  # of Route objects already in the system
  self.roots = []
  roots = fields[4].split(",")
+ rootOrder = 0
  for root in roots:
  route = None
  for check_route in system.route_list:
@@ -845,6 +833,9 @@ def __init__(self,line,system,el):
  " in system " + system.systemname + '.')
  else:
  self.roots.append(route)
+ # save order of route in connected route
+ route.rootOrder = rootOrder
+ rootOrder += 1
  if len(self.roots) < 1:
  el.add_error("No roots in _con.csv line [" + line + "]")
  # will be computed for routes in active systems later
@@ -961,8 +952,10 @@ def __init__(self,travelername,systems,route_hash,path="../../../UserData/list_f
  continue
  fields = re.split(' +',line)
  if len(fields) != 4:
- self.log_entries.append("Incorrect format line: " + line)
- continue
+ # OK if 5th field exists and starts with #
+ if len(fields) < 5 or not fields[4].startswith("#"):
+ self.log_entries.append("Incorrect format line: " + line)
+ continue
 
  # find the root that matches in some system and when we do, match labels
  route_entry = fields[1].lower()
@@ -1873,7 +1866,7 @@ def __init__(self,filename,descr,vertices,edges,format,category):
  help="path to the user list file data")
 parser.add_argument("-d", "--databasename", default="TravelMapping", \
  help="Database name for .sql file name")
-parser.add_argument("-l", "--logfilepath", default=".", help="Path to write log files")
+parser.add_argument("-l", "--logfilepath", default=".", help="Path to write log files, which should have a \"users\" subdirectory")
 parser.add_argument("-c", "--csvstatfilepath", default=".", help="Path to write csv statistics files")
 parser.add_argument("-g", "--graphfilepath", default=".", help="Path to write graph format data files")
 parser.add_argument("-k", "--skipgraphs", action="store_true", help="Turn off generation of graph files")
@@ -1996,43 +1989,40 @@ def __init__(self,filename,descr,vertices,edges,format,category):
 
 # check for duplicate root entries among Route and ConnectedRoute
 # data in all highway systems
-print(et.et() + "Checking for duplicate list names in routes, roots in routes and connected routes.",end="",flush=True)
-roots = []
-list_names = []
+print(et.et() + "Checking for duplicate list names in routes, roots in routes and connected routes.",flush=True)
+roots = set()
+list_names = set()
 duplicate_list_names = set()
 for h in highway_systems:
- print(".", end="",flush=True)
  for r in h.route_list:
  if r.root in roots:
  el.add_error("Duplicate root in route lists: " + r.root)
  else:
- roots.append(r.root)
+ roots.add(r.root)
  list_name = r.region + ' ' + r.list_entry_name()
  if list_name in list_names:
  duplicate_list_names.add(list_name)
  else:
- list_names.append(list_name)
+ list_names.add(list_name)
 
-con_roots = []
+con_roots = set()
 for h in highway_systems:
- print(".", end="",flush=True)
  for r in h.con_route_list:
  for cr in r.roots:
  if cr.root in con_roots:
  el.add_error("Duplicate root in con_route lists: " + cr.root)
  else:
- con_roots.append(cr.root)
-print("!", flush=True)
+ con_roots.add(cr.root)
 
 # Make sure every route was listed as a part of some connected route
 if len(roots) == len(con_roots):
  print("Check passed: same number of routes as connected route roots. " + str(len(roots)))
 else:
  el.add_error("Check FAILED: " + str(len(roots)) + " routes != " + str(len(con_roots)) + " connected route roots.")
- for r in con_roots:
- roots.remove(r)
+ roots = roots - con_roots
  # there will be some leftovers, let's look up their routes to make
- # an error report entry
+ # an error report entry (not worried about efficiency as there would
+ # only be a few in reasonable cases)
  num_found = 0
  for h in highway_systems:
  for r in h.route_list:
@@ -2952,7 +2942,7 @@ def run(self):
 # write log files for traveler lists
 print(et.et() + "Writing traveler list logs.",flush=True)
 for t in traveler_lists:
- t.write_log(args.logfilepath)
+ t.write_log(args.logfilepath+"/users")
 
 # write stats csv files
 print(et.et() + "Writing stats csv files.",flush=True)
@@ -3297,40 +3287,46 @@ def run(self):
  # color for its mapping, a level (one of active, preview, devel), and
  # a boolean indicating if the system is active for mapping in the
  # project in the field 'active'
- sqlfile.write('CREATE TABLE systems (systemName VARCHAR(10), countryCode CHAR(3), fullName VARCHAR(60), color VARCHAR(16), level VARCHAR(10), tier INTEGER, PRIMARY KEY(systemName));\n')
+ sqlfile.write('CREATE TABLE systems (systemName VARCHAR(10), countryCode CHAR(3), fullName VARCHAR(60), color VARCHAR(16), level VARCHAR(10), tier INTEGER, csvOrder INTEGER, PRIMARY KEY(systemName));\n')
  sqlfile.write('INSERT INTO systems VALUES\n')
  first = True
+ csvOrder = 0
  for h in highway_systems:
  if not first:
  sqlfile.write(",")
  first = False
  sqlfile.write("('" + h.systemname + "','" + h.country + "','" +
  h.fullname + "','" + h.color + "','" + h.level +
- "','" + str(h.tier) + "')\n")
+ "','" + str(h.tier) + "','" + str(csvOrder) + "')\n")
+ csvOrder += 1
  sqlfile.write(";\n")
 
  # next, a table of highways, with the same fields as in the first line
- sqlfile.write('CREATE TABLE routes (systemName VARCHAR(10), region VARCHAR(8), route VARCHAR(16), banner VARCHAR(6), abbrev VARCHAR(3), city VARCHAR(100), root VARCHAR(32), mileage FLOAT, PRIMARY KEY(root), FOREIGN KEY (systemName) REFERENCES systems(systemName));\n')
+ sqlfile.write('CREATE TABLE routes (systemName VARCHAR(10), region VARCHAR(8), route VARCHAR(16), banner VARCHAR(6), abbrev VARCHAR(3), city VARCHAR(100), root VARCHAR(32), mileage FLOAT, rootOrder INTEGER, csvOrder INTEGER, PRIMARY KEY(root), FOREIGN KEY (systemName) REFERENCES systems(systemName));\n')
  sqlfile.write('INSERT INTO routes VALUES\n')
  first = True
+ csvOrder = 0
  for h in highway_systems:
  for r in h.route_list:
  if not first:
  sqlfile.write(",")
  first = False
- sqlfile.write("(" + r.csv_line() + ")\n")
+ sqlfile.write("(" + r.csv_line() + ",'" + str(csvOrder) + "')\n")
+ csvOrder += 1
  sqlfile.write(";\n")
 
  # connected routes table, but only first "root" in each in this table
- sqlfile.write('CREATE TABLE connectedRoutes (systemName VARCHAR(10), route VARCHAR(16), banner VARCHAR(6), groupName VARCHAR(100), firstRoot VARCHAR(32), mileage FLOAT, PRIMARY KEY(firstRoot), FOREIGN KEY (firstRoot) REFERENCES routes(root));\n')
+ sqlfile.write('CREATE TABLE connectedRoutes (systemName VARCHAR(10), route VARCHAR(16), banner VARCHAR(6), groupName VARCHAR(100), firstRoot VARCHAR(32), mileage FLOAT, csvOrder INTEGER, PRIMARY KEY(firstRoot), FOREIGN KEY (firstRoot) REFERENCES routes(root));\n')
  sqlfile.write('INSERT INTO connectedRoutes VALUES\n')
  first = True
+ csvOrder = 0
  for h in highway_systems:
  for cr in h.con_route_list:
  if not first:
  sqlfile.write(",")
  first = False
- sqlfile.write("(" + cr.csv_line() + ")\n")
+ sqlfile.write("(" + cr.csv_line() + ",'" + str(csvOrder) + "')\n")
+ csvOrder += 1
  sqlfile.write(";\n")
 
  # This table has remaining roots for any connected route

siteupdate/python-teresco/siteupdate.sh

@@ -5,10 +5,10 @@ read_data=1
 transfer=1
 logdir=logs
 statdir=stats
-graphdir=graphs
+graphdir=graphdata
 nmpmerged=nmp_merged
 graphflag=
-mkdir -p $logdir $statdir $graphdir
+mkdir -p $logdir/users $statdir $graphdir
 date
 if [ $# -eq 1 ]; then
  if [ "$1" == "--noread" ]; then
@@ -24,7 +24,7 @@ fi
 if [ "$read_data" == "1" ]; then
  echo "siteupdate.sh: launching siteupdate.py"
  echo "siteupdate.sh: cleaning $logdir and $statdir"
- /bin/rm -f $logdir/*.log $statdir/*.csv 
+ /bin/rm -f $logdir/*.log $logdir/users/*.log $statdir/*.csv 
  # Add -k to prevent generation of new version of graphs
  # Remove -k to generate new version of graphs
  PYTHONIOENCODING='utf-8' ./siteupdate.py $graphflag -l $logdir -c $statdir -g $graphdir -n $nmpmerged | tee $logdir/siteupdate.log 2>&1 || exit 1

siteupdate/python-teresco/xferlogs.sh

@@ -4,13 +4,13 @@
 # and can run concurrently with parts of siteupdate.sh
 #
 echo "xferlogs.sh: Creating logstoxfer.tar"
-tar cf logstoxfer.tar $1/*.log $2/*.csv $3/*.tmg $3/*.nmp
+tar cf logstoxfer.tar $1/*.log $1/users/*.log $2/*.csv $3/*.tmg $1/*.nmp
 echo "xferlogs.sh: Bzipping logstoxfer.tar"
 bzip2 -9f logstoxfer.tar
 echo "xferlogs.sh: Transfering logstoxfer.tar.bz2"
 scp logstoxfer.tar.bz2 blizzard.teresco.org:/tmp
 echo "xferlogs.sh: Launching command to bunzip and extract logstoxfer.tar.bz2 on blizzard"
 ssh blizzard.teresco.org "cd /home/www/tm; /bin/rm -rf ${1} ${2}; bzcat /tmp/logstoxfer.tar.bz2 | tar xpf -"
 echo "xferlogs.sh: Launching command to create zip graph archive on blizzard"
-ssh blizzard.teresco.org "cd /home/www/tm; zip -q graphs.zip graphs/*.tmg"
+ssh blizzard.teresco.org "cd /home/www/tm; zip -q graphs.zip ${3}/*.tmg"
 echo "xferlogs.sh: complete"