sync, 2018-09-21

nmpfilter/README.md

@@ -0,0 +1,10 @@
+# nmpfilter
+
+**Purpose:**<br>
+Splits *tm-master.nmp* into smaller .nmp files filtered by region.
+
+**usage:**<br>
+`nmpfilter HwyDataDir MasterNMP OutputDir`, where
+* `HwyDataDir` is the path of the *hwy_data* directory in the HighwayData repository, or equivalent
+* `MasterNMP` is the path of *tm-master.nmp*
+* `OutputDir` is the directory in which to write the resulting .nmp files. Trailing slash required.

nmpfilter/nmpfilter.cpp

@@ -0,0 +1,69 @@
+// Travel Mapping Project, Eric Bryant, 2018
+#include <dirent.h>
+#include <fstream>
+#include <iostream>
+#include <string>
+#include <vector>
+using namespace std;
+
+bool LwrCaseValid(string &RgStr)
+// Check whether a string is a valid region; convert to lower case
+{	if (RgStr == ".") return 0;
+	if (RgStr == "..") return 0;
+	if (RgStr == "_systems")	return 0;
+	if (RgStr == "_boundaries")	return 0;
+	for (unsigned int i = 0; i < RgStr.size(); i++)
+ if (RgStr[i] >= 'A' && RgStr[i] <= 'Z')
+ RgStr[i] += 32;
+	return 1;
+}
+
+void GetRegions(char *HwyDataDir, vector<string> &RgList)
+// Get a list of regions by scanning subdirectories of HwyData/
+{	DIR *dir;
+	dirent *ent;
+	if ((dir = opendir (HwyDataDir)) != NULL)
+	{	while ((ent = readdir (dir)) != NULL)
+ {	string RgStr = ent->d_name;
+ if (LwrCaseValid(RgStr)) RgList.push_back(RgStr);
+ }
+ closedir(dir);
+	}
+	else cout << "Error opening directory " << HwyDataDir << ". (Not found?)\n";
+}
+
+string GetRootRg(string RgStr)
+// Remove dashes from region codes, to match region strings as found in Roots
+{	for (int dash = RgStr.find('-'); dash > 0; dash = RgStr.find('-'))
+ RgStr.erase(dash, 1);
+	return RgStr;
+}
+
+void filter(vector<string> &RgList, char *MasterNMP, char *OutputDir)
+// Write .nmp files for each region
+{	for (unsigned int i = 0; i < RgList.size(); i++)
+	{	ifstream master(MasterNMP);
+ if (!master)
+ {	std::cout << MasterNMP << " not found\n";
+ return;
+ }
+ string pt1, pt2;
+ string outfile = OutputDir+RgList[i]+".nmp";
+ ofstream nmp(outfile.data());
+ while (getline(master, pt1) && getline(master, pt2))
+ {	string RootRg = GetRootRg(RgList[i]);
+ if (pt1.substr(0, pt1.find('.')) == RootRg || pt2.substr(0, pt2.find('.')) == RootRg)
+ nmp << pt1 << '\n' << pt2 << '\n';
+ }
+	}
+}
+
+int main(int argc, char *argv[])
+{	if (argc != 4)
+	{	cout << "usage: nmpfilter HwyDataDir MasterNMP OutputDir\n";
+ return 0;
+	}
+	vector<string> RgList;
+	GetRegions(argv[1], RgList);
+	filter(RgList, argv[2], argv[3]);
+}

siteupdate/python-teresco/localupdate.sh

@@ -38,7 +38,7 @@ if [ "$pull" == "1" ]; then
 fi
 
 echo "$0: creating directories"
-mkdir -p $datestr/$logdir/users $datestr/$statdir $datestr/$nmpmdir
+mkdir -p $datestr/$logdir/users $datestr/$statdir $datestr/$nmpmdir $datestr/$logdir/nmpbyregion
 if [ "$graphflag" != "-k" ]; then
  mkdir -p $datestr/$graphdir
 fi
@@ -47,10 +47,29 @@ echo "$0: launching siteupdate.py"
 PYTHONIOENCODING='utf-8' ./siteupdate.py -d TravelMapping-$datestr $graphflag -l $datestr/$logdir -c $datestr/$statdir -g $datestr/$graphdir -n $datestr/$nmpmdir | tee $datestr/$logdir/siteupdate.log 2>&1 || exit 1
 date
 
+if [ -x ../../nmpfilter/nmpfilter ]; then
+ echo "$0: running nmpfilter"
+ ../../nmpfilter/nmpfilter $tmbase/HighwayData/hwy_data $datestr/$logdir/tm-master.nmp $datestr/$logdir/nmpbyregion/
+ echo "$0: creating zip archive of all nmp files created by nmpfilter"
+ cd $datestr/$logdir/nmpbyregion
+ zip -q nmpbyregion.zip *.nmp
+ cd -
+else
+ echo "$0: SKIPPING nmpfilter (../../nmpfilter/nmpfilter not executable)"
+fi
+
 if [ "$install" == "0" ]; then
  echo "$0: SKIPPING file copies and DB update"
  exit 0
 fi
+
+if [ "$graphflag" != "-k" ]; then
+ echo "$0: creating zip archive of all graphs"
+ cd $datestr/$graphdir
+ zip -q graphs.zip *.tmg
+ cd -
+fi
+
 echo "$0: installing logs, stats, nmp_merged, graphs, archiving old contents in $tmpdir/$datestr"
 mkdir -p $tmpdir/$datestr
 mv $tmwebbase/$logdir $tmpdir/$datestr
@@ -68,9 +87,11 @@ echo "$0: switching to DB copy"
 ln -sf $tmwebbase/lib/tm.conf.updating $tmwebbase/lib/tm.conf
 touch $tmwebbase/dbupdating
 echo "$0: loading primary DB"
+date
 mysql --defaults-group-suffix=tmapadmin -u travmapadmin TravelMapping < TravelMapping-$datestr.sql
 /bin/rm $tmwebbase/dbupdating
 echo "$0: switching to primary DB"
+date
 ln -sf $tmwebbase/lib/tm.conf.standard $tmwebbase/lib/tm.conf
 echo "$0: loading DB copy"
 mysql --defaults-group-suffix=tmapadmin -u travmapadmin TravelMappingCopy < TravelMapping-$datestr.sql