threaded label hashes & connectivity checks

siteupdate/cplusplus/Makefile

@@ -22,6 +22,7 @@ CommonObjects = \
  classes/HighwaySystem/HighwaySystem.o \
  classes/Region/Region.o \
  classes/Route/Route.o \
+ classes/Route/label_and_connect.o \
  classes/TravelerList/TravelerList.o \
  classes/Waypoint/Waypoint.o \
  classes/Waypoint/canonical_waypoint_name/canonical_waypoint_name.o \

siteupdate/cplusplus/classes/Route/Route.cpp

@@ -84,7 +84,8 @@ Route::Route(std::string &line, HighwaySystem *sys, ErrorList &el, std::unordere
  el.add_error("Duplicate root in " + system->systemname + ".csv: " + root +
  " already in " + root_hash.at(root)->system->systemname + ".csv");
 	// insert list name into pri_list_hash, checking for duplicate .list names
-	std::string list_name = upper(readable_name());
+	std::string list_name(readable_name());
+	upper(list_name.data());
 	if (alt_list_hash.find(list_name) != alt_list_hash.end())
  el.add_error("Duplicate main list name in " + root + ": '" + readable_name() +
  "' already points to " + alt_list_hash.at(list_name)->root);
@@ -93,7 +94,8 @@ Route::Route(std::string &line, HighwaySystem *sys, ErrorList &el, std::unordere
  "' already points to " + pri_list_hash.at(list_name)->root);
 	// insert alt names into alt_list_hash, checking for duplicate .list names
 	for (std::string& a : alt_route_names)
-	{ list_name = upper(rg_str + ' ' + a);
+	{ list_name = rg_str + ' ' + a;
+ upper(list_name.data());
  if (pri_list_hash.find(list_name) != pri_list_hash.end())
  el.add_error("Duplicate alt route name in " + root + ": '" + region->code + ' ' + a +
  "' already points to " + pri_list_hash.at(list_name)->root);

siteupdate/cplusplus/classes/Route/Route.h

@@ -93,6 +93,7 @@ class Route
 	void write_nmp_merged(std::string);
 	void store_traveled_segments(TravelerList*, std::ofstream &, unsigned int, unsigned int);
 	void compute_stats_r();
+	void label_and_connect(ErrorList&);
 	Waypoint* con_beg();
 	Waypoint* con_end();
 };

siteupdate/cplusplus/classes/Route/label_and_connect.cpp

@@ -0,0 +1,79 @@
+#include "Route.h"
+#include "../ConnectedRoute/ConnectedRoute.h"
+#include "../DatacheckEntry/DatacheckEntry.h"
+#include "../ErrorList/ErrorList.h"
+#include "../HighwaySystem/HighwaySystem.h"
+#include "../Waypoint/Waypoint.h"
+#include "../../functions/upper.h"
+
+void Route::label_and_connect(ErrorList& el)
+{	// check for unconnected chopped routes
+	if (!con_route)
+	{	el.add_error(system->systemname + ".csv: root " + root + " not matched by any connected route root.");
+ return;
+	}
+
+	// check for mismatched route endpoints within connected routes
+	#define q con_route->roots[rootOrder-1]
+	if ( rootOrder > 0 && q->point_list.size() > 1 && point_list.size() > 1 && !con_beg()->same_coords(q->con_end()) )
+	{	if ( q->con_beg()->same_coords(con_beg()) )
+ {	//std::cout << "DEBUG: marking only " << q->str() << " reversed" << std::endl;
+ //if (q->is_reversed) std::cout << "DEBUG: " << q->str() << " already reversed!" << std::endl;
+ q->is_reversed = 1;
+ }
+ else if ( q->con_end()->same_coords(con_end()) )
+ {	//std::cout << "DEBUG: marking only " << str() << " reversed" << std::endl;
+ //if ( is_reversed) std::cout << "DEBUG: " << str() << " already reversed!" << std::endl;
+ is_reversed = 1;
+ }
+ else if ( q->con_beg()->same_coords(con_end()) )
+ {	//std::cout << "DEBUG: marking both " << q->str() << " and " << str() << " reversed" << std::endl;
+ //if (q->is_reversed) std::cout << "DEBUG: " << q->str() << " already reversed!" << std::endl;
+ //if ( is_reversed) std::cout << "DEBUG: " << str() << " already reversed!" << std::endl;
+ q->is_reversed = 1;
+ is_reversed = 1;
+ }
+ else
+ {	DatacheckEntry::add(this, con_beg()->label, "", "",
+ "DISCONNECTED_ROUTE", q->con_end()->root_at_label());
+ DatacheckEntry::add(q, q->con_end()->label, "", "",
+ "DISCONNECTED_ROUTE", con_beg()->root_at_label());
+ }
+	}
+	#undef q
+
+	// create label hashes and check for duplicates
+	for (unsigned int index = 0; index < point_list.size(); index++)
+	{	// ignore case and leading '+' or '*'
+ const char* lbegin = point_list[index]->label.data();
+ while (*lbegin == '+' || *lbegin == '*') lbegin++;
+ std::string upper_label(lbegin);
+ upper(upper_label.data());
+ // if primary label not duplicated, add to pri_label_hash
+ if (alt_label_hash.find(upper_label) != alt_label_hash.end())
+ {	DatacheckEntry::add(this, point_list[index]->label, "", "", "DUPLICATE_LABEL", "");
+ duplicate_labels.insert(upper_label);
+ }
+ else if (!pri_label_hash.insert(std::pair<std::string, unsigned int>(upper_label, index)).second)
+ {	DatacheckEntry::add(this, point_list[index]->label, "", "", "DUPLICATE_LABEL", "");
+ duplicate_labels.insert(upper_label);
+ }
+ for (std::string& a : point_list[index]->alt_labels)
+ {	// create canonical AltLabels
+ while (a[0] == '+' || a[0] == '*') a = a.data()+1;
+ upper(a.data());
+ // populate unused set
+ unused_alt_labels.insert(a);
+ // create label->index hashes and check if AltLabels duplicated
+ std::unordered_map<std::string, unsigned int>::iterator A = pri_label_hash.find(a);
+ if (A != pri_label_hash.end())
+ {	DatacheckEntry::add(this, point_list[A->second]->label, "", "", "DUPLICATE_LABEL", "");
+ duplicate_labels.insert(a);
+ }
+ else if (!alt_label_hash.insert(std::pair<std::string, unsigned int>(a, index)).second)
+ {	DatacheckEntry::add(this, a, "", "", "DUPLICATE_LABEL", "");
+ duplicate_labels.insert(a);
+ }
+ }
+	}
+}

siteupdate/cplusplus/classes/TravelerList/mark_chopped_route_segments.cpp

@@ -1,5 +1,6 @@
 // find the route that matches and when we do, match labels
-std::string lookup = upper(fields[0]) + ( ' ' + upper(std::string(fields[1])) ); // leave fields[1] intact for potential AltRouteName note
+std::string lookup = std::string(fields[0]) + ' ' + fields[1];
+upper(lookup.data());
 // look for region/route combo, first in pri_list_hash
 std::unordered_map<std::string,Route*>::iterator rit = Route::pri_list_hash.find(lookup);
 // and then if not found, in alt_list_hash

siteupdate/cplusplus/classes/TravelerList/mark_connected_route_segments.cpp

@@ -1,5 +1,7 @@
-std::string lookup1 = upper(std::string(fields[0]) + ' ' + fields[1]); // leave fields intact for potential AltRouteName note
-std::string lookup2 = upper(std::string(fields[3]) + ' ' + fields[4]); // leave fields intact for potential AltRouteName note
+std::string lookup1 = std::string(fields[0]) + ' ' + fields[1];
+std::string lookup2 = std::string(fields[3]) + ' ' + fields[4];
+upper(lookup1.data());
+upper(lookup2.data());
 // look for region/route combos, first in pri_list_hash
 std::unordered_map<std::string,Route*>::iterator rit1 = Route::pri_list_hash.find(lookup1);
 std::unordered_map<std::string,Route*>::iterator rit2 = Route::pri_list_hash.find(lookup2);

siteupdate/cplusplus/functions/lower.cpp

@@ -1,11 +1,5 @@
 #include "lower.h"
 
-std::string lower(std::string str)
-{	for (unsigned int c = 0; c < str.size(); c++)
- if (str[c] >= 'A' && str[c] <= 'Z') str[c] += 32;
-	return str;
-}
-
 const char *lower(const char *str)
 {	for (char* c = (char*)str; *c != 0; c++)
  if (*c >= 'A' && *c <= 'Z') *c += 32;

siteupdate/cplusplus/functions/lower.h

@@ -1,4 +1,2 @@
 #include <string>
-
-std::string lower(std::string);
 const char *lower(const char*);

siteupdate/cplusplus/functions/upper.cpp

@@ -1,11 +1,5 @@
 #include "upper.h"
 
-std::string upper(std::string str)
-{	for (unsigned int c = 0; c < str.size(); c++)
- if (str[c] >= 'a' && str[c] <= 'z') str[c] -= 32;
-	return str;
-}
-
 const char *upper(const char *str)
 {	for (char* c = (char*)str; *c != 0; c++)
  if (*c >= 'a' && *c <= 'z') *c -= 32;

siteupdate/cplusplus/functions/upper.h

@@ -1,4 +1,2 @@
 #include <string>
-
-std::string upper(std::string);
 const char *upper(const char*);

siteupdate/cplusplus/siteupdate.cpp

@@ -257,7 +257,6 @@ int main(int argc, char *argv[])
 	cout << et.et() << "Searching for near-miss points." << endl;
  #ifdef threading_enabled
 	hs_it = highway_systems.begin();
-
 	THREADLOOP thr[t] = thread(NmpSearchThread, t, &highway_systems, &hs_it, &list_mtx, &all_waypoints);
 	THREADLOOP thr[t].join();
  #else
@@ -319,78 +318,15 @@ int main(int argc, char *argv[])
 	#include "tasks/concurrency_detection.cpp"
 
 	cout << et.et() << "Processing waypoint labels and checking for unconnected chopped routes." << endl;
-	for (HighwaySystem* h : highway_systems)
+ #ifdef threading_enabled
+	hs_it = highway_systems.begin();
+	THREADLOOP thr[t] = thread(LabelConThread, t, &highway_systems, &hs_it, &list_mtx, &el);
+	THREADLOOP thr[t].join();
+ #else
+	for (HighwaySystem *h : highway_systems)
  for (Route* r : h->route_list)
- {	// check for unconnected chopped routes
- if (!r->con_route)
- {	el.add_error(r->system->systemname + ".csv: root " + r->root + " not matched by any connected route root.");
- continue;
- }
-
- // check for mismatched route endpoints within connected routes
- #define q r->con_route->roots[r->rootOrder-1]
- if ( r->rootOrder > 0 && q->point_list.size() > 1 && r->point_list.size() > 1 && !r->con_beg()->same_coords(q->con_end()) )
- {	if ( q->con_beg()->same_coords(r->con_beg()) )
- {	//std::cout << "DEBUG: marking only " << q->str() << " reversed" << std::endl;
- //if (q->is_reversed) std::cout << "DEBUG: " << q->str() << " already reversed!" << std::endl;
- q->is_reversed = 1;
- }
- else if ( q->con_end()->same_coords(r->con_end()) )
- {	//std::cout << "DEBUG: marking only " << r->str() << " reversed" << std::endl;
- //if (r->is_reversed) std::cout << "DEBUG: " << r->str() << " already reversed!" << std::endl;
- r->is_reversed = 1;
- }
- else if ( q->con_beg()->same_coords(r->con_end()) )
- {	//std::cout << "DEBUG: marking both " << q->str() << " and " << r->str() << " reversed" << std::endl;
- //if (q->is_reversed) std::cout << "DEBUG: " << q->str() << " already reversed!" << std::endl;
- //if (r->is_reversed) std::cout << "DEBUG: " << r->str() << " already reversed!" << std::endl;
- q->is_reversed = 1;
- r->is_reversed = 1;
- }
- else
- {	DatacheckEntry::add(r, r->con_beg()->label, "", "",
- "DISCONNECTED_ROUTE", q->con_end()->root_at_label());
- DatacheckEntry::add(q, q->con_end()->label, "", "",
- "DISCONNECTED_ROUTE", r->con_beg()->root_at_label());
- }
- }
- #undef q
-
- // create label hashes and check for duplicates
- #define w r->point_list[index]
- for (unsigned int index = 0; index < r->point_list.size(); index++)
- {	// ignore case and leading '+' or '*'
- std::string upper_label = upper(w->label);
- while (upper_label[0] == '+' || upper_label[0] == '*')
- upper_label = upper_label.substr(1);
- // if primary label not duplicated, add to r->pri_label_hash
- if (r->alt_label_hash.find(upper_label) != r->alt_label_hash.end())
- {	DatacheckEntry::add(r, upper_label, "", "", "DUPLICATE_LABEL", "");
- r->duplicate_labels.insert(upper_label);
- }
- else if (!r->pri_label_hash.insert(std::pair<std::string, unsigned int>(upper_label, index)).second)
- {	DatacheckEntry::add(r, upper_label, "", "", "DUPLICATE_LABEL", "");
- r->duplicate_labels.insert(upper_label);
- }
- for (std::string& a : w->alt_labels)
- {	// create canonical AltLabels
- while (a[0] == '+' || a[0] == '*') a = a.substr(1);
- upper(a.data());
- // populate unused set
- r->unused_alt_labels.insert(a);
- // create label->index hashes and check if AltLabels duplicated
- if (r->pri_label_hash.find(a) != r->pri_label_hash.end())
- {	DatacheckEntry::add(r, a, "", "", "DUPLICATE_LABEL", "");
- r->duplicate_labels.insert(a);
- }
- else if (!r->alt_label_hash.insert(std::pair<std::string, unsigned int>(a, index)).second)
- {	DatacheckEntry::add(r, a, "", "", "DUPLICATE_LABEL", "");
- r->duplicate_labels.insert(a);
- }
- }
- }
- #undef w
- }
+ r->label_and_connect(el);
+ #endif
 
 	#include "tasks/read_updates.cpp"
 

siteupdate/cplusplus/threads/LabelConThread.cpp

@@ -0,0 +1,19 @@
+void LabelConThread
+(	unsigned int id, std::list<HighwaySystem*>* hs_list,
+	std::list<HighwaySystem*>::iterator* it, std::mutex* hs_mtx, ErrorList* el
+)
+{	//printf("Starting LabelConThread %02i\n", id); fflush(stdout);
+	while (*it != hs_list->end())
+	{	hs_mtx->lock();
+ if (*it == hs_list->end())
+ {	hs_mtx->unlock();
+ return;
+ }
+ HighwaySystem *h(**it);
+ //printf("LabelConThread %02i assigned %s\n", id, h->systemname.data()); fflush(stdout);
+ (*it)++;
+ //printf("LabelConThread %02i (*it)++\n", id); fflush(stdout);
+ hs_mtx->unlock();
+ for (Route* r : h->route_list) r->label_and_connect(*el);
+	}
+}

siteupdate/cplusplus/threads/threads.cpp

@@ -12,6 +12,7 @@
 #include "CompStatsRThread.cpp"
 #include "CompStatsTThread.cpp"
 #include "ConcAugThread.cpp"
+#include "LabelConThread.cpp"
 #include "MasterTmgThread.cpp"
 #include "NmpMergedThread.cpp"
 #include "NmpSearchThread.cpp"

siteupdate/cplusplus/threads/threads.h

@@ -16,6 +16,7 @@ class WaypointQuadtree;
 void CompStatsRThread(unsigned int, std::list<HighwaySystem*>*, std::list<HighwaySystem*>::iterator*, std::mutex*);
 void CompStatsTThread(unsigned int, std::list<TravelerList *>*, std::list<TravelerList *>::iterator*, std::mutex*);
 void ConcAugThread (unsigned int, std::list<TravelerList *>*, std::list<TravelerList *>::iterator*, std::mutex*, std::list<std::string>*);
+void LabelConThread (unsigned int, std::list<HighwaySystem*>*, std::list<HighwaySystem*>::iterator*, std::mutex*, ErrorList*);
 void MasterTmgThread(HighwayGraph*, std::vector<GraphListEntry>*, std::string, std::list<TravelerList*>*, size_t*, std::mutex*, std::mutex*, WaypointQuadtree*, ElapsedTime*);
 void NmpMergedThread (unsigned int, std::list<HighwaySystem*>*, std::list<HighwaySystem*>::iterator*, std::mutex*, std::string*);
 void NmpSearchThread (unsigned int, std::list<HighwaySystem*>*, std::list<HighwaySystem*>::iterator*, std::mutex*, WaypointQuadtree*);

siteupdate/python-teresco/siteupdate.py

@@ -2950,10 +2950,10 @@ def run(self):
  upper_label = w.label.lstrip('+*').upper()
  # if primary label not duplicated, add to r.pri_label_hash
  if upper_label in r.alt_label_hash:
- datacheckerrors.append(DatacheckEntry(r, [upper_label], "DUPLICATE_LABEL"))
+ datacheckerrors.append(DatacheckEntry(r, [w.label], "DUPLICATE_LABEL"))
  r.duplicate_labels.add(upper_label)
  elif upper_label in r.pri_label_hash:
- datacheckerrors.append(DatacheckEntry(r, [upper_label], "DUPLICATE_LABEL"))
+ datacheckerrors.append(DatacheckEntry(r, [w.label], "DUPLICATE_LABEL"))
  r.duplicate_labels.add(upper_label)
  else:
  r.pri_label_hash[upper_label] = index
@@ -2964,7 +2964,7 @@ def run(self):
  r.unused_alt_labels.add(w.alt_labels[a])
  # create label->index hashes and check if AltLabels duplicated
  if w.alt_labels[a] in r.pri_label_hash:
- datacheckerrors.append(DatacheckEntry(r, [w.alt_labels[a]], "DUPLICATE_LABEL"))
+ datacheckerrors.append(DatacheckEntry(r, [point_list[r.pri_label_hash[a]].label], "DUPLICATE_LABEL"))
  r.duplicate_labels.add(w.alt_labels[a])
  elif w.alt_labels[a] in r.alt_label_hash:
  datacheckerrors.append(DatacheckEntry(r, [w.alt_labels[a]], "DUPLICATE_LABEL"))