From f2cbdec0b2972fde89567564d1db1ac5cff1b8fc Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Sat, 8 Sep 2018 03:58:41 -0400 Subject: [PATCH 1/4] new program to filter tm-master.nmp by region --- nmpfilter/README.md | 10 ++++++ nmpfilter/nmpfilter.cpp | 69 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 nmpfilter/README.md create mode 100644 nmpfilter/nmpfilter.cpp diff --git a/nmpfilter/README.md b/nmpfilter/README.md new file mode 100644 index 00000000..32e01966 --- /dev/null +++ b/nmpfilter/README.md @@ -0,0 +1,10 @@ +# nmpfilter + +**Purpose:**
+Splits *tm-master.nmp* into smaller .nmp files filtered by region. + +**usage:**
+`nmpfilter HwyDataDir MasterNMP OutputDir`, where +* `HwyDataDir` is the path of the *hwy_data* directory in the HighwayData repository, or equivalent +* `MasterNMP` is the path of *tm-master.nmp* +* `OutputDir` is the directory in which to write the resulting .nmp files. Trailing slash required. diff --git a/nmpfilter/nmpfilter.cpp b/nmpfilter/nmpfilter.cpp new file mode 100644 index 00000000..89d26ccc --- /dev/null +++ b/nmpfilter/nmpfilter.cpp @@ -0,0 +1,69 @@ +// Travel Mapping Project, Eric Bryant, 2018 +#include +#include +#include +#include +#include +using namespace std; + +bool LwrCaseValid(string &RgStr) +// Check whether a string is a valid region; convert to lower case +{ if (RgStr == ".") return 0; + if (RgStr == "..") return 0; + if (RgStr == "_systems") return 0; + if (RgStr == "_boundaries") return 0; + for (unsigned int i = 0; i < RgStr.size(); i++) + if (RgStr[i] >= 'A' && RgStr[i] <= 'Z') + RgStr[i] += 32; + return 1; +} + +void GetRegions(char *HwyDataDir, vector &RgList) +// Get a list of regions by scanning subdirectories of HwyData/ +{ DIR *dir; + dirent *ent; + if ((dir = opendir (HwyDataDir)) != NULL) + { while ((ent = readdir (dir)) != NULL) + { string RgStr = ent->d_name; + if (LwrCaseValid(RgStr)) RgList.push_back(RgStr); + } + closedir(dir); + } + else cout << "Error opening directory " << HwyDataDir << ". (Not found?)\n"; +} + +string GetRootRg(string RgStr) +// Remove dashes from region codes, to match region strings as found in Roots +{ for (int dash = RgStr.find('-'); dash > 0; dash = RgStr.find('-')) + RgStr.erase(dash, 1); + return RgStr; +} + +void filter(vector &RgList, char *MasterNMP, char *OutputDir) +// Write .nmp files for each region +{ for (unsigned int i = 0; i < RgList.size(); i++) + { ifstream master(MasterNMP); + if (!master) + { std::cout << MasterNMP << " not found\n"; + return; + } + string pt1, pt2; + string outfile = OutputDir+RgList[i]+".nmp"; + ofstream nmp(outfile.data()); + while (getline(master, pt1) && getline(master, pt2)) + { string RootRg = GetRootRg(RgList[i]); + if (pt1.substr(0, pt1.find('.')) == RootRg || pt2.substr(0, pt2.find('.')) == RootRg) + nmp << pt1 << '\n' << pt2 << '\n'; + } + } +} + +int main(int argc, char *argv[]) +{ if (argc != 4) + { cout << "usage: nmpfilter HwyDataDir MasterNMP OutputDir\n"; + return 0; + } + vector RgList; + GetRegions(argv[1], RgList); + filter(RgList, argv[2], argv[3]); +} From d36028fbdc72f12922f6d407534ad541900daebf Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Sat, 8 Sep 2018 23:20:32 -0400 Subject: [PATCH 2/4] Add nmpfilter to localupdate.sh. --- siteupdate/python-teresco/localupdate.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/siteupdate/python-teresco/localupdate.sh b/siteupdate/python-teresco/localupdate.sh index 4ca1680d..ee53ea77 100644 --- a/siteupdate/python-teresco/localupdate.sh +++ b/siteupdate/python-teresco/localupdate.sh @@ -38,7 +38,7 @@ if [ "$pull" == "1" ]; then fi echo "$0: creating directories" -mkdir -p $datestr/$logdir/users $datestr/$statdir $datestr/$nmpmdir +mkdir -p $datestr/$logdir/users $datestr/$statdir $datestr/$nmpmdir $datestr/$logdir/nmpbyregion if [ "$graphflag" != "-k" ]; then mkdir -p $datestr/$graphdir fi @@ -47,6 +47,13 @@ echo "$0: launching siteupdate.py" PYTHONIOENCODING='utf-8' ./siteupdate.py -d TravelMapping-$datestr $graphflag -l $datestr/$logdir -c $datestr/$statdir -g $datestr/$graphdir -n $datestr/$nmpmdir | tee $datestr/$logdir/siteupdate.log 2>&1 || exit 1 date +if [ -x ../../nmpfilter/nmpfilter ]; then + echo "$0: running nmpfilter" + ../../nmpfilter/nmpfilter $tmbase/HighwayData/hwy_data $datestr/$logdir/tm-master.nmp $datestr/$logdir/nmpbyregion/ +else + echo "$0: SKIPPING nmpfilter (../../nmpfilter/nmpfilter not executable)" +fi + if [ "$install" == "0" ]; then echo "$0: SKIPPING file copies and DB update" exit 0 From 1d5522b9623ae84609d0b98fc0d7d8e58e3d4997 Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Mon, 10 Sep 2018 21:34:16 -0400 Subject: [PATCH 3/4] Implementation of #104 and site update part of https://github.com/TravelMapping/Web/issues/283 --- siteupdate/python-teresco/localupdate.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/siteupdate/python-teresco/localupdate.sh b/siteupdate/python-teresco/localupdate.sh index ee53ea77..a342a3a8 100644 --- a/siteupdate/python-teresco/localupdate.sh +++ b/siteupdate/python-teresco/localupdate.sh @@ -50,6 +50,10 @@ date if [ -x ../../nmpfilter/nmpfilter ]; then echo "$0: running nmpfilter" ../../nmpfilter/nmpfilter $tmbase/HighwayData/hwy_data $datestr/$logdir/tm-master.nmp $datestr/$logdir/nmpbyregion/ + echo "$0: creating zip archive of all nmp files created by nmpfilter" + cd $datestr/$logdir/nmpbyregion + zip nmpbyregion.zip *.nmp + cd - else echo "$0: SKIPPING nmpfilter (../../nmpfilter/nmpfilter not executable)" fi @@ -58,6 +62,14 @@ if [ "$install" == "0" ]; then echo "$0: SKIPPING file copies and DB update" exit 0 fi + +if [ "$graphflag" != "-k" ]; then + echo "$0: creating zip archive of all graphs" + cd $datestr/$graphdir + zip graphs.zip *.tmg + cd - +fi + echo "$0: installing logs, stats, nmp_merged, graphs, archiving old contents in $tmpdir/$datestr" mkdir -p $tmpdir/$datestr mv $tmwebbase/$logdir $tmpdir/$datestr From bf8238adfd1748a0423cf8082a31b0bac46c76b8 Mon Sep 17 00:00:00 2001 From: jteresco Date: Mon, 10 Sep 2018 23:08:48 -0400 Subject: [PATCH 4/4] Site update output tweaks. --- siteupdate/python-teresco/localupdate.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/siteupdate/python-teresco/localupdate.sh b/siteupdate/python-teresco/localupdate.sh index a342a3a8..a0e77ca4 100644 --- a/siteupdate/python-teresco/localupdate.sh +++ b/siteupdate/python-teresco/localupdate.sh @@ -52,7 +52,7 @@ if [ -x ../../nmpfilter/nmpfilter ]; then ../../nmpfilter/nmpfilter $tmbase/HighwayData/hwy_data $datestr/$logdir/tm-master.nmp $datestr/$logdir/nmpbyregion/ echo "$0: creating zip archive of all nmp files created by nmpfilter" cd $datestr/$logdir/nmpbyregion - zip nmpbyregion.zip *.nmp + zip -q nmpbyregion.zip *.nmp cd - else echo "$0: SKIPPING nmpfilter (../../nmpfilter/nmpfilter not executable)" @@ -66,7 +66,7 @@ fi if [ "$graphflag" != "-k" ]; then echo "$0: creating zip archive of all graphs" cd $datestr/$graphdir - zip graphs.zip *.tmg + zip -q graphs.zip *.tmg cd - fi @@ -87,9 +87,11 @@ echo "$0: switching to DB copy" ln -sf $tmwebbase/lib/tm.conf.updating $tmwebbase/lib/tm.conf touch $tmwebbase/dbupdating echo "$0: loading primary DB" +date mysql --defaults-group-suffix=tmapadmin -u travmapadmin TravelMapping < TravelMapping-$datestr.sql /bin/rm $tmwebbase/dbupdating echo "$0: switching to primary DB" +date ln -sf $tmwebbase/lib/tm.conf.standard $tmwebbase/lib/tm.conf echo "$0: loading DB copy" mysql --defaults-group-suffix=tmapadmin -u travmapadmin TravelMappingCopy < TravelMapping-$datestr.sql