From b93077289c88bb09429d59683b27044011d438bc Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Fri, 21 Sep 2018 03:54:47 -0400 Subject: [PATCH 01/17] Rename nmpfilter.cpp to nmpbyregion.cpp --- nmpfilter/{nmpfilter.cpp => nmpbyregion.cpp} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename nmpfilter/{nmpfilter.cpp => nmpbyregion.cpp} (100%) diff --git a/nmpfilter/nmpfilter.cpp b/nmpfilter/nmpbyregion.cpp similarity index 100% rename from nmpfilter/nmpfilter.cpp rename to nmpfilter/nmpbyregion.cpp From 65158646c713a7f80bf0da25d126288bd4eff30d Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Fri, 21 Sep 2018 03:55:39 -0400 Subject: [PATCH 02/17] nmpfilter -> nmpbyregion --- nmpfilter/README.md | 4 ++-- nmpfilter/nmpbyregion.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nmpfilter/README.md b/nmpfilter/README.md index 32e01966..05fd3d99 100644 --- a/nmpfilter/README.md +++ b/nmpfilter/README.md @@ -1,10 +1,10 @@ -# nmpfilter +# nmpbyregion **Purpose:**
Splits *tm-master.nmp* into smaller .nmp files filtered by region. **usage:**
-`nmpfilter HwyDataDir MasterNMP OutputDir`, where +`nmpbyregion HwyDataDir MasterNMP OutputDir`, where * `HwyDataDir` is the path of the *hwy_data* directory in the HighwayData repository, or equivalent * `MasterNMP` is the path of *tm-master.nmp* * `OutputDir` is the directory in which to write the resulting .nmp files. Trailing slash required. diff --git a/nmpfilter/nmpbyregion.cpp b/nmpfilter/nmpbyregion.cpp index 89d26ccc..64481b55 100644 --- a/nmpfilter/nmpbyregion.cpp +++ b/nmpfilter/nmpbyregion.cpp @@ -60,7 +60,7 @@ void filter(vector &RgList, char *MasterNMP, char *OutputDir) int main(int argc, char *argv[]) { if (argc != 4) - { cout << "usage: nmpfilter HwyDataDir MasterNMP OutputDir\n"; + { cout << "usage: nmpbyregion HwyDataDir MasterNMP OutputDir\n"; return 0; } vector RgList; From 4b4da292f1d7a8262d10aa18b0bd9a22900219f4 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Fri, 21 Sep 2018 04:01:26 -0400 Subject: [PATCH 03/17] localupdate.sh: nmpfilter -> nmpbyregion --- siteupdate/python-teresco/localupdate.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/siteupdate/python-teresco/localupdate.sh b/siteupdate/python-teresco/localupdate.sh index a0e77ca4..8dcd9c88 100644 --- a/siteupdate/python-teresco/localupdate.sh +++ b/siteupdate/python-teresco/localupdate.sh @@ -47,15 +47,15 @@ echo "$0: launching siteupdate.py" PYTHONIOENCODING='utf-8' ./siteupdate.py -d TravelMapping-$datestr $graphflag -l $datestr/$logdir -c $datestr/$statdir -g $datestr/$graphdir -n $datestr/$nmpmdir | tee $datestr/$logdir/siteupdate.log 2>&1 || exit 1 date -if [ -x ../../nmpfilter/nmpfilter ]; then - echo "$0: running nmpfilter" - ../../nmpfilter/nmpfilter $tmbase/HighwayData/hwy_data $datestr/$logdir/tm-master.nmp $datestr/$logdir/nmpbyregion/ - echo "$0: creating zip archive of all nmp files created by nmpfilter" +if [ -x ../../nmpfilter/nmpbyregion ]; then + echo "$0: running nmpbyregion" + ../../nmpfilter/nmpbyregion $tmbase/HighwayData/hwy_data $datestr/$logdir/tm-master.nmp $datestr/$logdir/nmpbyregion/ + echo "$0: creating zip archive of all nmp files created by nmpbyregion" cd $datestr/$logdir/nmpbyregion zip -q nmpbyregion.zip *.nmp cd - else - echo "$0: SKIPPING nmpfilter (../../nmpfilter/nmpfilter not executable)" + echo "$0: SKIPPING nmpbyregion (../../nmpfilter/nmpbyregion not executable)" fi if [ "$install" == "0" ]; then From c4e6e46e8b1dd66704428717e33637e00b5331ab Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Sun, 30 Sep 2018 14:13:43 -0400 Subject: [PATCH 04/17] nmpbycountry, nmpfpfilter, nmplifilter --- nmpfilter/README.md | 35 ++++++++++++ nmpfilter/nmpbycountry.cpp | 109 +++++++++++++++++++++++++++++++++++++ nmpfilter/nmpfpfilter.cpp | 23 ++++++++ nmpfilter/nmplifilter.cpp | 23 ++++++++ 4 files changed, 190 insertions(+) create mode 100644 nmpfilter/nmpbycountry.cpp create mode 100644 nmpfilter/nmpfpfilter.cpp create mode 100644 nmpfilter/nmplifilter.cpp diff --git a/nmpfilter/README.md b/nmpfilter/README.md index 05fd3d99..ae670548 100644 --- a/nmpfilter/README.md +++ b/nmpfilter/README.md @@ -1,3 +1,18 @@ +# nmpbycountry + +**Purpose:**
+Splits *tm-master.nmp* into smaller .nmp files filtered by country. + +**usage:**
+`nmpbycountry RgCsvFile MasterNMP OutputDir`, where +* `RgCsvFile` is the path of *regions.csv* +* `MasterNMP` is the path of *tm-master.nmp* +* `OutputDir` is the directory in which to write the resulting .nmp files. Trailing slash required.
+ +The output directory should be empty before running *nmpbycountry*. Files for countries without NMPs will not be written. Thus any existing file for a country that subsequently has all NMPs removed will not be overwritten. + +--- + # nmpbyregion **Purpose:**
@@ -8,3 +23,23 @@ Splits *tm-master.nmp* into smaller .nmp files filtered by region. * `HwyDataDir` is the path of the *hwy_data* directory in the HighwayData repository, or equivalent * `MasterNMP` is the path of *tm-master.nmp* * `OutputDir` is the directory in which to write the resulting .nmp files. Trailing slash required. + +--- + +# nmpfpfilter + +**Purpose:**
+Removes marked false-positive pairs from a specified .nmp file. + +**usage:**
+`nmpfpfilter InputFile OutputFile` + +--- + +# nmplifilter + +**Purpose:**
+Removes all NMP pairs from a specified .nmp file except those that look intentional. + +**usage:**
+`nmplifilter InputFile OutputFile` diff --git a/nmpfilter/nmpbycountry.cpp b/nmpfilter/nmpbycountry.cpp new file mode 100644 index 00000000..09290a28 --- /dev/null +++ b/nmpfilter/nmpbycountry.cpp @@ -0,0 +1,109 @@ +// Travel Mapping Project, Eric Bryant, 2018 +#include +#include +#include +#include +#include +using namespace std; + +class region +{ public: + string code, name, country, continent, regionType; + + void UseRootRg() + { // Remove dashes from code + for (int dash = code.find('-'); dash > 0; dash = code.find('-')) + code.erase(dash, 1); + // Convert to lower case + for (unsigned int i = 0; i < code.size(); i++) + if (code[i] >= 'A' && code[i] <= 'Z') + code[i] += 32; + } + + region (string &CSVline) + { unsigned int i = 0; + while (i < CSVline.size() && CSVline[i] != ';') { code.push_back(CSVline[i]); i++; } i++; + while (i < CSVline.size() && CSVline[i] != ';') { name.push_back(CSVline[i]); i++; } i++; + while (i < CSVline.size() && CSVline[i] != ';') { country.push_back(CSVline[i]); i++; } i++; + while (i < CSVline.size() && CSVline[i] != ';') { continent.push_back(CSVline[i]); i++; } i++; + while (i < CSVline.size() && CSVline[i] != ';') { regionType.push_back(CSVline[i]); i++; } i++; + UseRootRg(); + } +}; + +bool GetRegions(char *CsvFile, list &CoList, vector &RgList) +// Read contents of regions.csv +{ ifstream CSV(CsvFile); + if (!CSV) { cout << CsvFile << " not found!\n"; return 0;} + + string CSVline; + getline(CSV, CSVline); // Skip header row + while (getline(CSV, CSVline)) + { RgList.emplace_back(CSVline); + CoList.emplace_back(RgList.back().country); + } + CoList.sort(); + list::iterator c = CoList.begin(); + list::iterator d = CoList.begin(); d++; + while (d != CoList.end()) // step thru each country in list + { while (*c == *d) d = CoList.erase(d); // remove duplicates + c++; d++; + } + return 1; +} + +bool ReadMaster(char *MasterNMP, vector &master) +// Read contents of tm-master.nmp +{ ifstream CSV(MasterNMP); + if (!CSV) + { std::cout << MasterNMP << " not found\n"; + return 0; + } + string CSVline; + while (getline(CSV, CSVline)) master.emplace_back(CSVline); + return 1; +} + +bool RgMatch(vector &RgList, string &co, string rg) +// Find region code in RgList vector; check whether it matches country +{ for (unsigned int r = 0; r < RgList.size(); r++) + { if (RgList[r].code == rg) + if (RgList[r].country == co) return 1; + else return 0; + } + return 0; +} + +void filter(list &CoList, vector &RgList, vector &master, char *OutputDir) +// Write .nmp files for each country +{ for (list::iterator c = CoList.begin(); c != CoList.end(); c++) + { vector output; + for (unsigned int l = 1; l < master.size(); l+=2) + { if ( RgMatch(RgList, *c, master[l-1].substr(0, master[l-1].find('.'))) \ + || RgMatch(RgList, *c, master[l].substr(0, master[l].find('.'))) ) + { output.emplace_back(master[l-1]); + output.emplace_back(master[l]); + } + } + //cout << "output vector created: " << output.size() << " lines.\n"; + if (output.size()) + { string outfile = OutputDir+*c+".nmp"; + ofstream nmp(outfile.data()); + for (unsigned int l = 0; l < output.size(); l++) + nmp << output[l] << '\n'; + } + } +} + +int main(int argc, char *argv[]) +{ if (argc != 4) + { cout << "usage: nmpbycountry RgCsvFile MasterNMP OutputDir\n"; + return 0; + } + list CoList; + vector RgList; + vector master; + if (!GetRegions(argv[1], CoList, RgList)) return 0; + if (!ReadMaster(argv[2], master)) return 0; + filter(CoList, RgList, master, argv[3]); +} diff --git a/nmpfilter/nmpfpfilter.cpp b/nmpfilter/nmpfpfilter.cpp new file mode 100644 index 00000000..f682407a --- /dev/null +++ b/nmpfilter/nmpfpfilter.cpp @@ -0,0 +1,23 @@ +// Travel Mapping Project, Eric Bryant, 2018 +#include +#include +#include +using namespace std; + +int main(int argc, char *argv[]) +{ if (argc != 3) + { cout << "usage: nmpfpfilter InputFile OutputFile\n"; + return 0; + } + ifstream in(argv[1]); + if (!in) + { std::cout << argv[1] << " not found\n"; + return 0; + } + string pt1, pt2; + ofstream out(argv[2]); + while (getline(in, pt1) && getline(in, pt2)) + { if (pt1.substr(pt1.find_last_of(' ')+1, 2) != "FP" || pt2.substr(pt2.find_last_of(' ')+1, 2) != "FP") + out << pt1 << '\n' << pt2 << '\n'; + } +} diff --git a/nmpfilter/nmplifilter.cpp b/nmpfilter/nmplifilter.cpp new file mode 100644 index 00000000..6b16e500 --- /dev/null +++ b/nmpfilter/nmplifilter.cpp @@ -0,0 +1,23 @@ +// Travel Mapping Project, Eric Bryant, 2018 +#include +#include +#include +using namespace std; + +int main(int argc, char *argv[]) +{ if (argc != 3) + { cout << "usage: nmplifilter InputFile OutputFile\n"; + return 0; + } + ifstream in(argv[1]); + if (!in) + { std::cout << argv[1] << " not found\n"; + return 0; + } + string pt1, pt2; + ofstream out(argv[2]); + while (getline(in, pt1) && getline(in, pt2)) + { if (pt1.substr(pt1.size()-2, 2) == "LI" || pt2.substr(pt2.size()-2, 2) == "LI") + out << pt1 << '\n' << pt2 << '\n'; + } +} From 3357d325ceda2cd012450b3e984f17bbbb67899f Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Fri, 12 Oct 2018 12:29:48 -0400 Subject: [PATCH 05/17] 20181012, trailing zeros in vertex InfoBox --- tmg2html/tmg2html.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tmg2html/tmg2html.cpp b/tmg2html/tmg2html.cpp index 94381bce..aefbd82d 100644 --- a/tmg2html/tmg2html.cpp +++ b/tmg2html/tmg2html.cpp @@ -511,9 +511,9 @@ int main(int argc, char *argv[]) html << " \"
\" + vertex[vNum].label + \"\" +\n"; html << " \"
Vertex \" + vNum +\n"; html << " \"
Coords: \" +\n"; - html << " vertex[vNum].lat + \",\" + vertex[vNum].lon +\n"; + html << " vertex[vNum].lat.toFixed(6) + \"&lon=\" +\n"; + html << " vertex[vNum].lon.toFixed(6) + \"'>Coords: \" +\n"; + html << " vertex[vNum].lat.toFixed(6) + \",\" + vertex[vNum].lon.toFixed(6) +\n"; html << " \"\";\n"; html << " VertexMarker(vNum, \"#FF0000\");\n"; html << " selectedV = vNum;\n"; From f137a5e06082dc3df5f67cfd64cdc8bd2cbc0425 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Sat, 13 Oct 2018 15:08:45 -0400 Subject: [PATCH 06/17] 20181013, InfoBox position, size, overflow --- tmg2html/tmg2html.cpp | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tmg2html/tmg2html.cpp b/tmg2html/tmg2html.cpp index aefbd82d..8f223421 100644 --- a/tmg2html/tmg2html.cpp +++ b/tmg2html/tmg2html.cpp @@ -111,6 +111,15 @@ int main(int argc, char *argv[]) html << " { background-color:#D0D0D0;\n"; html << " }\n"; html << "\n"; + html << " #InfoBox\n"; + html << " { background-color:#D0D0D0;\n"; + html << " position:fixed;\n"; + html << " left: 700px;\n"; + html << " right: 0px;\n"; + html << " height: 89px;\n"; + html << " overflow:auto;\n"; + html << " }\n"; + html << "\n"; html << " #vInfo\n"; html << " { position:fixed;\n"; html << " left: 700px;\n"; @@ -147,10 +156,10 @@ int main(int argc, char *argv[]) html << "
\n"; html << "\n"; html << "
\n"; - html << "\n"; + html << "
\n"; html << "\n"; - html << "\n"; + html << " X\n"; + html << "\n"; html << "
Info\n"; - html << " X
\n"; html << "
\n"; html << "\n"; @@ -359,7 +368,7 @@ int main(int argc, char *argv[]) html << "function EdgeSelect(eNum)\n"; html << "{ ClearInfo();\n"; html << " document.getElementById(\"InfoBox\").innerHTML =\n"; - html << " \"
\" + edge[eNum].label + \"\" +\n"; + html << " \"\" + edge[eNum].label + \"\" +\n"; html << " \"
Edge \" + eNum;\n"; html << " EdgeHighlight(eNum);\n"; html << " selectedE = eNum;\n"; @@ -508,7 +517,7 @@ int main(int argc, char *argv[]) html << "function VertexSelect(vNum)\n"; html << "{ ClearInfo();\n"; html << " document.getElementById(\"InfoBox\").innerHTML =\n"; - html << " \"
\" + vertex[vNum].label + \"\" +\n"; + html << " \"\" + vertex[vNum].label + \"\" +\n"; html << " \"
Vertex \" + vNum +\n"; html << " \"
0: - print("ABORTING due to " + str(len(el.error_list)) + " errors:") - for i in range(len(el.error_list)): - print(str(i+1) + ": " + el.error_list[i]) - sys.exit(1) - -# perform most datachecks here (list initialized above) -for h in highway_systems: - print(".",end="",flush=True) - for r in h.route_list: - # set to be used per-route to find label duplicates - all_route_labels = set() - # set of tuples to be used for finding duplicate coordinates - coords_used = set() - - visible_distance = 0.0 - # note that we assume the first point will be visible in each route - # so the following is simply a placeholder - last_visible = None - prev_w = None - - # look for hidden termini - if r.point_list[0].is_hidden: - labels = [] - labels.append(r.point_list[0].label) - datacheckerrors.append(DatacheckEntry(r,labels,'HIDDEN_TERMINUS')) - if r.point_list[len(r.point_list)-1].is_hidden: - labels = [] - labels.append(r.point_list[len(r.point_list)-1].label) - datacheckerrors.append(DatacheckEntry(r,labels,'HIDDEN_TERMINUS')) - - for w in r.point_list: - # duplicate labels - label_list = w.alt_labels.copy() - label_list.append(w.label) - for label in label_list: - lower_label = label.lower().strip("+*") - if lower_label in all_route_labels: - labels = [] - labels.append(lower_label) - datacheckerrors.append(DatacheckEntry(r,labels,"DUPLICATE_LABEL")) - else: - all_route_labels.add(lower_label) - - # out-of-bounds coords - if w.lat > 90 or w.lat < -90 or w.lng > 180 or w.lng < -180: - labels = [] - labels.append(w.label) - datacheckerrors.append(DatacheckEntry(r,labels,'OUT_OF_BOUNDS', - "("+str(w.lat)+","+str(w.lng)+")")) - - # duplicate coordinates - latlng = w.lat, w.lng - if latlng in coords_used: - for other_w in r.point_list: - if w == other_w: - break - if w.lat == other_w.lat and w.lng == other_w.lng and w.label != other_w.label: - labels = [] - labels.append(other_w.label) - labels.append(w.label) - datacheckerrors.append(DatacheckEntry(r,labels,"DUPLICATE_COORDS", - "("+str(latlng[0])+","+str(latlng[1])+")")) - else: - coords_used.add(latlng) - - # visible distance update, and last segment length check - if prev_w is not None: - last_distance = w.distance_to(prev_w) - visible_distance += last_distance - if last_distance > 20.0: - labels = [] - labels.append(prev_w.label) - labels.append(w.label) - datacheckerrors.append(DatacheckEntry(r,labels,'LONG_SEGMENT', - "{0:.2f}".format(last_distance))) - - if not w.is_hidden: - # complete visible distance check, omit report for active - # systems to reduce clutter - if visible_distance > 10.0 and not h.active(): - labels = [] - labels.append(last_visible.label) - labels.append(w.label) - datacheckerrors.append(DatacheckEntry(r,labels,'VISIBLE_DISTANCE', - "{0:.2f}".format(visible_distance))) - last_visible = w - visible_distance = 0.0 - - # looking for the route within the label - #match_start = w.label.find(r.route) - #if match_start >= 0: - # we have a potential match, just need to make sure if the route - # name ends with a number that the matched substring isn't followed - # by more numbers (e.g., NY50 is an OK label in NY5) - # if len(r.route) + match_start == len(w.label) or \ - # not w.label[len(r.route) + match_start].isdigit(): - # partially complete "references own route" -- too many FP - #or re.fullmatch('.*/'+r.route+'.*',w.label[w.label) : - # first check for number match after a slash, if there is one - selfref_found = False - if '/' in w.label and r.route[-1].isdigit(): - digit_starts = len(r.route)-1 - while digit_starts >= 0 and r.route[digit_starts].isdigit(): - digit_starts-=1 - if w.label[w.label.index('/')+1:] == r.route[digit_starts+1:]: - selfref_found = True - if w.label[w.label.index('/')+1:] == r.route: - selfref_found = True - if '_' in w.label[w.label.index('/')+1:] and w.label[w.label.index('/')+1:w.label.rindex('_')] == r.route[digit_starts+1:]: - selfref_found = True - if '_' in w.label[w.label.index('/')+1:] and w.label[w.label.index('/')+1:w.label.rindex('_')] == r.route: - selfref_found = True - - # now the remaining checks - if selfref_found or r.route+r.banner == w.label or re.fullmatch(r.route+r.banner+'[_/].*',w.label): - labels = [] - labels.append(w.label) - datacheckerrors.append(DatacheckEntry(r,labels,'LABEL_SELFREF')) - - # look for too many underscores in label - if w.label.count('_') > 1: - labels = [] - labels.append(w.label) - datacheckerrors.append(DatacheckEntry(r,labels,'LABEL_UNDERSCORES')) - - # look for too many characters after underscore in label - if '_' in w.label: - if w.label.index('_') < len(w.label) - 5: - labels = [] - labels.append(w.label) - datacheckerrors.append(DatacheckEntry(r,labels,'LONG_UNDERSCORE')) - - # look for too many slashes in label - if w.label.count('/') > 1: - labels = [] - labels.append(w.label) - datacheckerrors.append(DatacheckEntry(r,labels,'LABEL_SLASHES')) - - # look for parenthesis balance in label - if w.label.count('(') != w.label.count(')'): - labels = [] - labels.append(w.label) - datacheckerrors.append(DatacheckEntry(r,labels,'LABEL_PARENS')) - - # look for labels with invalid characters - if not re.fullmatch('[a-zA-Z0-9()/\+\*_\-\.]+', w.label): - labels = [] - labels.append(w.label) - datacheckerrors.append(DatacheckEntry(r,labels,'LABEL_INVALID_CHAR')) - - # look for labels with a slash after an underscore - if '_' in w.label and '/' in w.label and \ - w.label.index('/') > w.label.index('_'): - labels = [] - labels.append(w.label) - datacheckerrors.append(DatacheckEntry(r,labels,'NONTERMINAL_UNDERSCORE')) - - # look for I-xx with Bus instead of BL or BS - if re.fullmatch('I\-[0-9]*Bus', w.label): - labels = [] - labels.append(w.label) - datacheckerrors.append(DatacheckEntry(r,labels,'BUS_WITH_I')) - - # look for labels that look like hidden waypoints but - # which aren't hidden - if re.fullmatch('X[0-9][0-9][0-9][0-9][0-9][0-9]', w.label): - labels = [] - labels.append(w.label) - datacheckerrors.append(DatacheckEntry(r,labels,'LABEL_LOOKS_HIDDEN')) - - # look for USxxxA but not USxxxAlt, B/Bus (others?) - ##if re.fullmatch('US[0-9]+A.*', w.label) and not re.fullmatch('US[0-9]+Alt.*', w.label) or \ - ## re.fullmatch('US[0-9]+B.*', w.label) and \ - ## not (re.fullmatch('US[0-9]+Bus.*', w.label) or re.fullmatch('US[0-9]+Byp.*', w.label)): - ## labels = [] - ## labels.append(w.label) - ## datacheckerrors.append(DatacheckEntry(r,labels,'US_BANNER')) - - prev_w = w - - # angle check is easier with a traditional for loop and array indices - for i in range(1, len(r.point_list)-1): - #print("computing angle for " + str(r.point_list[i-1]) + ' ' + str(r.point_list[i]) + ' ' + str(r.point_list[i+1])) - if r.point_list[i-1].same_coords(r.point_list[i]) or \ - r.point_list[i+1].same_coords(r.point_list[i]): - labels = [] - labels.append(r.point_list[i-1].label) - labels.append(r.point_list[i].label) - labels.append(r.point_list[i+1].label) - datacheckerrors.append(DatacheckEntry(r,labels,'BAD_ANGLE')) - else: - angle = r.point_list[i].angle(r.point_list[i-1],r.point_list[i+1]) - if angle > 135: - labels = [] - labels.append(r.point_list[i-1].label) - labels.append(r.point_list[i].label) - labels.append(r.point_list[i+1].label) - datacheckerrors.append(DatacheckEntry(r,labels,'SHARP_ANGLE', - "{0:.2f}".format(angle))) -print("!", flush=True) -print(et.et() + "Found " + str(len(datacheckerrors)) + " datacheck errors.") - -# now mark false positives -print(et.et() + "Marking datacheck false positives.",end="",flush=True) -fpfile = open(args.logfilepath+'/nearmatchfps.log','w',encoding='utf-8') -fpfile.write("Log file created at: " + str(datetime.datetime.now()) + "\n") -toremove = [] -counter = 0 -for d in datacheckerrors: - #print("Checking: " + str(d)) - counter += 1 - if counter % 1000 == 0: - print(".", end="",flush=True) - for fp in datacheckfps: - #print("Comparing: " + str(d) + " to " + str(fp)) - if d.match(fp): - #print("Match!") - d.fp = True - toremove.append(fp) - break - if d.match_except_info(fp): - fpfile.write("DCERROR: " + str(d) + "\n") - fpfile.write("FPENTRY: " + fp[0] + ';' + fp[1] + ';' + fp[2] + ';' + fp[3] + ';' + fp[4] + ';' + fp[5] + '\n') - fpfile.write("REPLACEWITH: " + fp[0] + ';' + fp[1] + ';' + fp[2] + ';' + fp[3] + ';' + fp[4] + ';' + d.info + '\n') - -fpfile.close() -# now remove the ones we matched from the list -for fp in toremove: - counter += 1 - if counter % 1000 == 0: - print(".", end="",flush=True) - if fp in datacheckfps: - datacheckfps.remove(fp) - else: - print("Matched FP entry not in list!: " + str(fp)) -print("!", flush=True) - -# write log of unmatched false positives from the datacheckfps.csv -print(et.et() + "Writing log of unmatched datacheck FP entries.") -fpfile = open(args.logfilepath+'/unmatchedfps.log','w',encoding='utf-8') -fpfile.write("Log file created at: " + str(datetime.datetime.now()) + "\n") -if len(datacheckfps) > 0: - for entry in datacheckfps: - fpfile.write(entry[0] + ';' + entry[1] + ';' + entry[2] + ';' + entry[3] + ';' + entry[4] + ';' + entry[5] + '\n') -else: - fpfile.write("No unmatched FP entries.") -fpfile.close() - # Create hash table for faster lookup of routes by list file name print(et.et() + "Creating route hash table for list processing:",flush=True) route_hash = dict() From 73347c3fe6f72abf050ccf0cb94e2e57b61c86a0 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Mon, 29 Oct 2018 00:46:31 -0400 Subject: [PATCH 09/17] ...to after graph generation commit 2 of 2, for human-readable DIFFs --- siteupdate/python-teresco/siteupdate.py | 272 ++++++++++++++++++++++++ 1 file changed, 272 insertions(+) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index bb3a7088..c26cf07b 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2949,6 +2949,278 @@ def run(self): 'These graphs contain the routes on a continent.']) print("!") +# data check: visit each system and route and check for various problems +print(et.et() + "Performing data checks.",end="",flush=True) +# first, read in the false positives list +with open(args.highwaydatapath+"/datacheckfps.csv", "rt",encoding='utf-8') as file: + lines = file.readlines() + +lines.pop(0) # ignore header line +datacheckfps = [] +datacheck_always_error = [ 'DUPLICATE_LABEL', 'HIDDEN_TERMINUS', + 'LABEL_INVALID_CHAR', 'LABEL_SLASHES', + 'LONG_UNDERSCORE', 'NONTERMINAL_UNDERSCORE', + 'OUT_OF_BOUNDS' ] +for line in lines: + fields = line.rstrip('\n').split(';') + if len(fields) != 6: + el.add_error("Could not parse datacheckfps.csv line: " + line) + continue + if fields[4] in datacheck_always_error: + print("datacheckfps.csv line not allowed (always error): " + line) + continue + datacheckfps.append(fields) + +# See if we have any errors that should be fatal to the site update process +if len(el.error_list) > 0: + print("ABORTING due to " + str(len(el.error_list)) + " errors:") + for i in range(len(el.error_list)): + print(str(i+1) + ": " + el.error_list[i]) + sys.exit(1) + +# perform most datachecks here (list initialized above) +for h in highway_systems: + print(".",end="",flush=True) + for r in h.route_list: + # set to be used per-route to find label duplicates + all_route_labels = set() + # set of tuples to be used for finding duplicate coordinates + coords_used = set() + + visible_distance = 0.0 + # note that we assume the first point will be visible in each route + # so the following is simply a placeholder + last_visible = None + prev_w = None + + # look for hidden termini + if r.point_list[0].is_hidden: + labels = [] + labels.append(r.point_list[0].label) + datacheckerrors.append(DatacheckEntry(r,labels,'HIDDEN_TERMINUS')) + if r.point_list[len(r.point_list)-1].is_hidden: + labels = [] + labels.append(r.point_list[len(r.point_list)-1].label) + datacheckerrors.append(DatacheckEntry(r,labels,'HIDDEN_TERMINUS')) + + for w in r.point_list: + # duplicate labels + label_list = w.alt_labels.copy() + label_list.append(w.label) + for label in label_list: + lower_label = label.lower().strip("+*") + if lower_label in all_route_labels: + labels = [] + labels.append(lower_label) + datacheckerrors.append(DatacheckEntry(r,labels,"DUPLICATE_LABEL")) + else: + all_route_labels.add(lower_label) + + # out-of-bounds coords + if w.lat > 90 or w.lat < -90 or w.lng > 180 or w.lng < -180: + labels = [] + labels.append(w.label) + datacheckerrors.append(DatacheckEntry(r,labels,'OUT_OF_BOUNDS', + "("+str(w.lat)+","+str(w.lng)+")")) + + # duplicate coordinates + latlng = w.lat, w.lng + if latlng in coords_used: + for other_w in r.point_list: + if w == other_w: + break + if w.lat == other_w.lat and w.lng == other_w.lng and w.label != other_w.label: + labels = [] + labels.append(other_w.label) + labels.append(w.label) + datacheckerrors.append(DatacheckEntry(r,labels,"DUPLICATE_COORDS", + "("+str(latlng[0])+","+str(latlng[1])+")")) + else: + coords_used.add(latlng) + + # visible distance update, and last segment length check + if prev_w is not None: + last_distance = w.distance_to(prev_w) + visible_distance += last_distance + if last_distance > 20.0: + labels = [] + labels.append(prev_w.label) + labels.append(w.label) + datacheckerrors.append(DatacheckEntry(r,labels,'LONG_SEGMENT', + "{0:.2f}".format(last_distance))) + + if not w.is_hidden: + # complete visible distance check, omit report for active + # systems to reduce clutter + if visible_distance > 10.0 and not h.active(): + labels = [] + labels.append(last_visible.label) + labels.append(w.label) + datacheckerrors.append(DatacheckEntry(r,labels,'VISIBLE_DISTANCE', + "{0:.2f}".format(visible_distance))) + last_visible = w + visible_distance = 0.0 + + # looking for the route within the label + #match_start = w.label.find(r.route) + #if match_start >= 0: + # we have a potential match, just need to make sure if the route + # name ends with a number that the matched substring isn't followed + # by more numbers (e.g., NY50 is an OK label in NY5) + # if len(r.route) + match_start == len(w.label) or \ + # not w.label[len(r.route) + match_start].isdigit(): + # partially complete "references own route" -- too many FP + #or re.fullmatch('.*/'+r.route+'.*',w.label[w.label) : + # first check for number match after a slash, if there is one + selfref_found = False + if '/' in w.label and r.route[-1].isdigit(): + digit_starts = len(r.route)-1 + while digit_starts >= 0 and r.route[digit_starts].isdigit(): + digit_starts-=1 + if w.label[w.label.index('/')+1:] == r.route[digit_starts+1:]: + selfref_found = True + if w.label[w.label.index('/')+1:] == r.route: + selfref_found = True + if '_' in w.label[w.label.index('/')+1:] and w.label[w.label.index('/')+1:w.label.rindex('_')] == r.route[digit_starts+1:]: + selfref_found = True + if '_' in w.label[w.label.index('/')+1:] and w.label[w.label.index('/')+1:w.label.rindex('_')] == r.route: + selfref_found = True + + # now the remaining checks + if selfref_found or r.route+r.banner == w.label or re.fullmatch(r.route+r.banner+'[_/].*',w.label): + labels = [] + labels.append(w.label) + datacheckerrors.append(DatacheckEntry(r,labels,'LABEL_SELFREF')) + + # look for too many underscores in label + if w.label.count('_') > 1: + labels = [] + labels.append(w.label) + datacheckerrors.append(DatacheckEntry(r,labels,'LABEL_UNDERSCORES')) + + # look for too many characters after underscore in label + if '_' in w.label: + if w.label.index('_') < len(w.label) - 5: + labels = [] + labels.append(w.label) + datacheckerrors.append(DatacheckEntry(r,labels,'LONG_UNDERSCORE')) + + # look for too many slashes in label + if w.label.count('/') > 1: + labels = [] + labels.append(w.label) + datacheckerrors.append(DatacheckEntry(r,labels,'LABEL_SLASHES')) + + # look for parenthesis balance in label + if w.label.count('(') != w.label.count(')'): + labels = [] + labels.append(w.label) + datacheckerrors.append(DatacheckEntry(r,labels,'LABEL_PARENS')) + + # look for labels with invalid characters + if not re.fullmatch('[a-zA-Z0-9()/\+\*_\-\.]+', w.label): + labels = [] + labels.append(w.label) + datacheckerrors.append(DatacheckEntry(r,labels,'LABEL_INVALID_CHAR')) + + # look for labels with a slash after an underscore + if '_' in w.label and '/' in w.label and \ + w.label.index('/') > w.label.index('_'): + labels = [] + labels.append(w.label) + datacheckerrors.append(DatacheckEntry(r,labels,'NONTERMINAL_UNDERSCORE')) + + # look for I-xx with Bus instead of BL or BS + if re.fullmatch('I\-[0-9]*Bus', w.label): + labels = [] + labels.append(w.label) + datacheckerrors.append(DatacheckEntry(r,labels,'BUS_WITH_I')) + + # look for labels that look like hidden waypoints but + # which aren't hidden + if re.fullmatch('X[0-9][0-9][0-9][0-9][0-9][0-9]', w.label): + labels = [] + labels.append(w.label) + datacheckerrors.append(DatacheckEntry(r,labels,'LABEL_LOOKS_HIDDEN')) + + # look for USxxxA but not USxxxAlt, B/Bus (others?) + ##if re.fullmatch('US[0-9]+A.*', w.label) and not re.fullmatch('US[0-9]+Alt.*', w.label) or \ + ## re.fullmatch('US[0-9]+B.*', w.label) and \ + ## not (re.fullmatch('US[0-9]+Bus.*', w.label) or re.fullmatch('US[0-9]+Byp.*', w.label)): + ## labels = [] + ## labels.append(w.label) + ## datacheckerrors.append(DatacheckEntry(r,labels,'US_BANNER')) + + prev_w = w + + # angle check is easier with a traditional for loop and array indices + for i in range(1, len(r.point_list)-1): + #print("computing angle for " + str(r.point_list[i-1]) + ' ' + str(r.point_list[i]) + ' ' + str(r.point_list[i+1])) + if r.point_list[i-1].same_coords(r.point_list[i]) or \ + r.point_list[i+1].same_coords(r.point_list[i]): + labels = [] + labels.append(r.point_list[i-1].label) + labels.append(r.point_list[i].label) + labels.append(r.point_list[i+1].label) + datacheckerrors.append(DatacheckEntry(r,labels,'BAD_ANGLE')) + else: + angle = r.point_list[i].angle(r.point_list[i-1],r.point_list[i+1]) + if angle > 135: + labels = [] + labels.append(r.point_list[i-1].label) + labels.append(r.point_list[i].label) + labels.append(r.point_list[i+1].label) + datacheckerrors.append(DatacheckEntry(r,labels,'SHARP_ANGLE', + "{0:.2f}".format(angle))) +print("!", flush=True) +print(et.et() + "Found " + str(len(datacheckerrors)) + " datacheck errors.") + +# now mark false positives +print(et.et() + "Marking datacheck false positives.",end="",flush=True) +fpfile = open(args.logfilepath+'/nearmatchfps.log','w',encoding='utf-8') +fpfile.write("Log file created at: " + str(datetime.datetime.now()) + "\n") +toremove = [] +counter = 0 +for d in datacheckerrors: + #print("Checking: " + str(d)) + counter += 1 + if counter % 1000 == 0: + print(".", end="",flush=True) + for fp in datacheckfps: + #print("Comparing: " + str(d) + " to " + str(fp)) + if d.match(fp): + #print("Match!") + d.fp = True + toremove.append(fp) + break + if d.match_except_info(fp): + fpfile.write("DCERROR: " + str(d) + "\n") + fpfile.write("FPENTRY: " + fp[0] + ';' + fp[1] + ';' + fp[2] + ';' + fp[3] + ';' + fp[4] + ';' + fp[5] + '\n') + fpfile.write("REPLACEWITH: " + fp[0] + ';' + fp[1] + ';' + fp[2] + ';' + fp[3] + ';' + fp[4] + ';' + d.info + '\n') + +fpfile.close() +# now remove the ones we matched from the list +for fp in toremove: + counter += 1 + if counter % 1000 == 0: + print(".", end="",flush=True) + if fp in datacheckfps: + datacheckfps.remove(fp) + else: + print("Matched FP entry not in list!: " + str(fp)) +print("!", flush=True) + +# write log of unmatched false positives from the datacheckfps.csv +print(et.et() + "Writing log of unmatched datacheck FP entries.") +fpfile = open(args.logfilepath+'/unmatchedfps.log','w',encoding='utf-8') +fpfile.write("Log file created at: " + str(datetime.datetime.now()) + "\n") +if len(datacheckfps) > 0: + for entry in datacheckfps: + fpfile.write(entry[0] + ';' + entry[1] + ';' + entry[2] + ';' + entry[3] + ';' + entry[4] + ';' + entry[5] + '\n') +else: + fpfile.write("No unmatched FP entries.") +fpfile.close() + # datacheck.log file print(et.et() + "Writing datacheck.log") logfile = open(args.logfilepath + '/datacheck.log', 'w') From b6bd0e3b37e4bf5f94c68a2d61098b5f104da9f6 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Mon, 29 Oct 2018 00:56:32 -0400 Subject: [PATCH 10/17] always generate master graph -k or --skipgraphs only skips subgraphs --- siteupdate/python-teresco/siteupdate.py | 48 ++++++++++++------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index c26cf07b..8ebc5070 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2769,33 +2769,33 @@ def run(self): # Build a graph structure out of all highway data in active and # preview systems -if args.skipgraphs or args.errorcheck: - print(et.et() + "SKIPPING generation of graphs.", flush=True) -else: - print(et.et() + "Setting up for graphs of highway data.", flush=True) - graph_data = HighwayGraph(all_waypoints, highway_systems, datacheckerrors) - - print(et.et() + "Writing graph waypoint simplification log.", flush=True) - logfile = open(args.logfilepath + '/waypointsimplification.log', 'w') - for line in graph_data.waypoint_naming_log: - logfile.write(line + '\n') - logfile.close() - - # create list of graph information for the DB - graph_list = [] - graph_types = [] +print(et.et() + "Setting up for graphs of highway data.", flush=True) +graph_data = HighwayGraph(all_waypoints, highway_systems, datacheckerrors) + +print(et.et() + "Writing graph waypoint simplification log.", flush=True) +logfile = open(args.logfilepath + '/waypointsimplification.log', 'w') +for line in graph_data.waypoint_naming_log: + logfile.write(line + '\n') +logfile.close() + +# create list of graph information for the DB +graph_list = [] +graph_types = [] - # start generating graphs and making entries for graph DB table +# start generating graphs and making entries for graph DB table - print(et.et() + "Writing master TM simple graph file, tm-master-simple.tmg", flush=True) - (sv, se) = graph_data.write_master_tmg_simple(args.graphfilepath+'/tm-master-simple.tmg') - graph_list.append(GraphListEntry('tm-master-simple.tmg', 'All Travel Mapping Data', sv, se, 'simple', 'master')) - print(et.et() + "Writing master TM collapsed graph file, tm-master.tmg.", flush=True) - (cv, ce) = graph_data.write_master_tmg_collapsed(args.graphfilepath+'/tm-master.tmg') - graph_list.append(GraphListEntry('tm-master.tmg', 'All Travel Mapping Data', cv, ce, 'collapsed', 'master')) - graph_types.append(['master', 'All Travel Mapping Data', - 'These graphs contain all routes currently plotted in the Travel Mapping project.']) +print(et.et() + "Writing master TM simple graph file, tm-master-simple.tmg", flush=True) +(sv, se) = graph_data.write_master_tmg_simple(args.graphfilepath+'/tm-master-simple.tmg') +graph_list.append(GraphListEntry('tm-master-simple.tmg', 'All Travel Mapping Data', sv, se, 'simple', 'master')) +print(et.et() + "Writing master TM collapsed graph file, tm-master.tmg.", flush=True) +(cv, ce) = graph_data.write_master_tmg_collapsed(args.graphfilepath+'/tm-master.tmg') +graph_list.append(GraphListEntry('tm-master.tmg', 'All Travel Mapping Data', cv, ce, 'collapsed', 'master')) +graph_types.append(['master', 'All Travel Mapping Data', + 'These graphs contain all routes currently plotted in the Travel Mapping project.']) +if args.skipgraphs or args.errorcheck: + print(et.et() + "SKIPPING generation of subgraphs.", flush=True) +else: # graphs restricted by place/area - from areagraphs.csv file print(et.et() + "Creating area data graphs.", flush=True) with open(args.highwaydatapath+"/graphs/areagraphs.csv", "rt",encoding='utf-8') as file: From eac54655e4ef674bd2e75d4f1d9dc3dd9fe4412a Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Mon, 29 Oct 2018 03:38:37 -0400 Subject: [PATCH 11/17] move VISIBLE_HIDDEN_COLOC datacheck into graph gen to avoid multiple datacheck entries at same coordinates --- siteupdate/python-teresco/siteupdate.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 8ebc5070..375ae786 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -731,8 +731,6 @@ def read_wpt(self,all_waypoints,all_waypoints_lock,datacheckerrors,el,path="../. other_w.colocated = [ other_w ] other_w.colocated.append(w) w.colocated = other_w.colocated - if w.is_hidden != other_w.is_hidden: - datacheckerrors.append(DatacheckEntry(self,[w.label],"VISIBLE_HIDDEN_COLOC",other_w.route.root + "@" + other_w.label)) # look for near-miss points (before we add this one in) #print("DEBUG: START search for nmps for waypoint " + str(w) + " in quadtree of size " + str(all_waypoints.size())) @@ -1132,7 +1130,7 @@ class HighwayGraphVertexInfo: vertex. """ - def __init__(self,waypoint_list): + def __init__(self,waypoint_list,datacheckerrors): self.lat = waypoint_list[0].lat self.lng = waypoint_list[0].lng self.unique_name = waypoint_list[0].unique_name @@ -1150,11 +1148,22 @@ def __init__(self,waypoint_list): self.systems.add(w.route.system) self.incident_edges = [] self.incident_collapsed_edges = [] + # VISIBLE_HIDDEN_COLOC datacheck + if self.visible_hidden_coloc(waypoint_list): + datacheckerrors.append(DatacheckEntry(waypoint_list[0].route,[waypoint_list[0].label],"VISIBLE_HIDDEN_COLOC", + "("+str(waypoint_list[0].lat)+","+str(waypoint_list[0].lng)+")")) # printable string def __str__(self): return self.unique_name + @staticmethod + def visible_hidden_coloc(waypoint_list): + for w in range(len(waypoint_list)): + if waypoint_list[w].is_hidden != waypoint_list[0].is_hidden: + return True + return False + class HighwayGraphEdgeInfo: """This class encapsulates information needed for a 'standard' highway graph edge. @@ -1538,7 +1547,7 @@ def __init__(self, all_waypoints, highway_systems, datacheckerrors): # One copy of the vertices self.vertices = {} for label, pointlist in self.unique_waypoints.items(): - self.vertices[label] = HighwayGraphVertexInfo(pointlist) + self.vertices[label] = HighwayGraphVertexInfo(pointlist,datacheckerrors) # add edges, which end up in vertex adjacency lists, first one # copy for the full graph From 6912d340c7ec6dc79e644f3246c9ae5bb1744719 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Mon, 29 Oct 2018 16:33:08 -0400 Subject: [PATCH 12/17] deterministic VISIBLE_HIDDEN_COLOC entries deterministic route, waypoints, and info --- siteupdate/python-teresco/siteupdate.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 375ae786..b56a69c2 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1150,8 +1150,18 @@ def __init__(self,waypoint_list,datacheckerrors): self.incident_collapsed_edges = [] # VISIBLE_HIDDEN_COLOC datacheck if self.visible_hidden_coloc(waypoint_list): - datacheckerrors.append(DatacheckEntry(waypoint_list[0].route,[waypoint_list[0].label],"VISIBLE_HIDDEN_COLOC", - "("+str(waypoint_list[0].lat)+","+str(waypoint_list[0].lng)+")")) + # determine which route, label, and info to use for this entry asciibetically + vis_list = [] + hid_list = [] + for w in waypoint_list: + if w.is_hidden: + hid_list.append(w) + else: + vis_list.append(w) + vis_list.sort(key=lambda waypoint: waypoint.route.root + "@" + waypoint.label) + hid_list.sort(key=lambda waypoint: waypoint.route.root + "@" + waypoint.label) + datacheckerrors.append(DatacheckEntry(vis_list[0].route,[vis_list[0].label],"VISIBLE_HIDDEN_COLOC", + hid_list[0].route.root+"@"+hid_list[0].label)) # printable string def __str__(self): From e7f2f9d0fab1982396bb8e7901e37b8cc26bc0fd Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Mon, 29 Oct 2018 19:01:44 -0400 Subject: [PATCH 13/17] deterministic HIDDEN_JUNCTION entries only for > 2 incident_collapsed_edges, to avoid dual-reporting HIDDEN_TERMINUS entries --- siteupdate/python-teresco/siteupdate.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 375ae786..96aef925 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1577,8 +1577,11 @@ def __init__(self, all_waypoints, highway_systems, datacheckerrors): # compress edges adjacent to hidden vertices for label, vinfo in self.vertices.items(): if vinfo.is_hidden: - if len(vinfo.incident_collapsed_edges) != 2: - datacheckerrors.append(DatacheckEntry(vinfo.first_waypoint.route,[vinfo.unique_name],"HIDDEN_JUNCTION",str(len(vinfo.incident_collapsed_edges)))) + if len(vinfo.incident_collapsed_edges) < 2: + continue + if len(vinfo.incident_collapsed_edges) > 2: + dc_waypoint = sorted(vinfo.first_waypoint.colocated, key=lambda waypoint: waypoint.route.root + "@" + waypoint.label)[0] + datacheckerrors.append(DatacheckEntry(dc_waypoint.route,[dc_waypoint.label],"HIDDEN_JUNCTION",str(len(vinfo.incident_collapsed_edges)))) vinfo.is_hidden = False continue # construct from vertex_info this time From 3ffa26e6609fc50a14d1083d0832dde5221abdbf Mon Sep 17 00:00:00 2001 From: Jim Teresco Date: Wed, 31 Oct 2018 19:32:33 -0400 Subject: [PATCH 14/17] Reorder and add earning message about mysql user sleep processes. Temp improvement to deal with #111. --- siteupdate/python-teresco/localupdate.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/siteupdate/python-teresco/localupdate.sh b/siteupdate/python-teresco/localupdate.sh index 8dcd9c88..eee634c9 100644 --- a/siteupdate/python-teresco/localupdate.sh +++ b/siteupdate/python-teresco/localupdate.sh @@ -42,6 +42,18 @@ mkdir -p $datestr/$logdir/users $datestr/$statdir $datestr/$nmpmdir $datestr/$lo if [ "$graphflag" != "-k" ]; then mkdir -p $datestr/$graphdir fi +echo "$0: switching to DB copy" +ln -sf $tmwebbase/lib/tm.conf.updating $tmwebbase/lib/tm.conf +touch $tmwebbase/dbupdating +echo "**********************************************************************" +echo "**********************************************************************" +echo "* *" +echo "* CHECK FOR USER SLEEP MYSQL PROCESSES USING SHOW PROCESSLIST; *" +echo "* BEFORE THE SITE UPDATE SCRIPT FINISHES TO AVOID A POSSIBLE HANG *" +echo "* DURING INGESTION OF THE NEW .sql FILE. *" +echo "* *" +echo "**********************************************************************" +echo "**********************************************************************" echo "$0: launching siteupdate.py" PYTHONIOENCODING='utf-8' ./siteupdate.py -d TravelMapping-$datestr $graphflag -l $datestr/$logdir -c $datestr/$statdir -g $datestr/$graphdir -n $datestr/$nmpmdir | tee $datestr/$logdir/siteupdate.log 2>&1 || exit 1 @@ -83,9 +95,6 @@ if [ "$graphflag" != "-k" ]; then mv $datestr/$graphdir $tmwebbase fi rmdir $datestr -echo "$0: switching to DB copy" -ln -sf $tmwebbase/lib/tm.conf.updating $tmwebbase/lib/tm.conf -touch $tmwebbase/dbupdating echo "$0: loading primary DB" date mysql --defaults-group-suffix=tmapadmin -u travmapadmin TravelMapping < TravelMapping-$datestr.sql From b642e06489e6e51a482b9af991f704298e4501db Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Thu, 1 Nov 2018 22:35:48 -0400 Subject: [PATCH 15/17] unhide hidden termini in graphs --- siteupdate/python-teresco/siteupdate.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 96aef925..c04f07bc 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1578,6 +1578,8 @@ def __init__(self, all_waypoints, highway_systems, datacheckerrors): for label, vinfo in self.vertices.items(): if vinfo.is_hidden: if len(vinfo.incident_collapsed_edges) < 2: + # these cases are flagged as HIDDEN_TERMINUS + vinfo.is_hidden = False continue if len(vinfo.incident_collapsed_edges) > 2: dc_waypoint = sorted(vinfo.first_waypoint.colocated, key=lambda waypoint: waypoint.route.root + "@" + waypoint.label)[0] From 0b7f41c7e61c8422d520db11c0d908881794f525 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Sat, 3 Nov 2018 15:55:36 -0400 Subject: [PATCH 16/17] don't write master graph files when -k --- siteupdate/python-teresco/siteupdate.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 01f0c018..8ffe3902 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2808,18 +2808,18 @@ def run(self): # start generating graphs and making entries for graph DB table -print(et.et() + "Writing master TM simple graph file, tm-master-simple.tmg", flush=True) -(sv, se) = graph_data.write_master_tmg_simple(args.graphfilepath+'/tm-master-simple.tmg') -graph_list.append(GraphListEntry('tm-master-simple.tmg', 'All Travel Mapping Data', sv, se, 'simple', 'master')) -print(et.et() + "Writing master TM collapsed graph file, tm-master.tmg.", flush=True) -(cv, ce) = graph_data.write_master_tmg_collapsed(args.graphfilepath+'/tm-master.tmg') -graph_list.append(GraphListEntry('tm-master.tmg', 'All Travel Mapping Data', cv, ce, 'collapsed', 'master')) -graph_types.append(['master', 'All Travel Mapping Data', - 'These graphs contain all routes currently plotted in the Travel Mapping project.']) - if args.skipgraphs or args.errorcheck: print(et.et() + "SKIPPING generation of subgraphs.", flush=True) else: + print(et.et() + "Writing master TM simple graph file, tm-master-simple.tmg", flush=True) + (sv, se) = graph_data.write_master_tmg_simple(args.graphfilepath+'/tm-master-simple.tmg') + graph_list.append(GraphListEntry('tm-master-simple.tmg', 'All Travel Mapping Data', sv, se, 'simple', 'master')) + print(et.et() + "Writing master TM collapsed graph file, tm-master.tmg.", flush=True) + (cv, ce) = graph_data.write_master_tmg_collapsed(args.graphfilepath+'/tm-master.tmg') + graph_list.append(GraphListEntry('tm-master.tmg', 'All Travel Mapping Data', cv, ce, 'collapsed', 'master')) + graph_types.append(['master', 'All Travel Mapping Data', + 'These graphs contain all routes currently plotted in the Travel Mapping project.']) + # graphs restricted by place/area - from areagraphs.csv file print(et.et() + "Creating area data graphs.", flush=True) with open(args.highwaydatapath+"/graphs/areagraphs.csv", "rt",encoding='utf-8') as file: From 6a0bd275d31cff248ec1b9b9b799316fa0107220 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Sat, 3 Nov 2018 21:57:09 -0400 Subject: [PATCH 17/17] remove dead code --- siteupdate/python-teresco/siteupdate.py | 31 ------------------------- 1 file changed, 31 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 8ffe3902..0f47c36b 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -605,37 +605,6 @@ def set_segment_name(self): self.segment_name += "," self.segment_name += cs.route.list_entry_name() - def gra_line(self): - """compute the line that should represent this segment in a - .gra format graph file (2 waypoint numbers, edge label)""" - line = "" - if self.waypoint1.colocated is None: - line += str(self.waypoint1.vertex_num) + ' ' - else: - line += str(self.waypoint1.colocated[0].vertex_num) + ' ' - if self.waypoint2.colocated is None: - line += str(self.waypoint2.vertex_num) + ' ' - else: - line += str(self.waypoint2.colocated[0].vertex_num) + ' ' - line += self.segment_name - return line - - def edge_line_with_labels(self): - """compute the line that should represent this segment in an - alternate format for a potential future graph file format - (2 waypoint labels, edge label)""" - line = "" - if self.waypoint1.colocated is None: - line += str(self.waypoint1.unique_name) + ' ' - else: - line += str(self.waypoint1.colocated[0].unique_name) + ' ' - if self.waypoint2.colocated is None: - line += str(self.waypoint2.unique_name) + ' ' - else: - line += str(self.waypoint2.colocated[0].unique_name) + ' ' - line += self.segment_name - return line - class Route: """This class encapsulates the contents of one .csv file line that represents a highway within a system and the corresponding