Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
53 commits
Select commit Hold shift + click to select a range
2c75863
wpt whitespace/blank line bugfix
yakra Apr 25, 2020
14ea929
no strtok_mtx when reading waypoints
yakra Apr 25, 2020
34c407a
update comment: no more strtok
yakra Apr 26, 2020
e0c21d1
all_wpt_files: list -> set
yakra Apr 27, 2020
8c633d2
syntactic sugar
yakra Apr 27, 2020
61aa0f1
Merge pull request #317 from yakra/read_wpt
jteresco Apr 27, 2020
8bcc764
refactor roots & list names; smash case
yakra May 4, 2020
f50764d
range for loop cleanup
yakra May 14, 2020
5de9946
Merge pull request #320 from yakra/hash
jteresco May 14, 2020
11d0ca7
.list processing enhancements
yakra May 20, 2020
a24d047
Merge pull request #330 from yakra/TravelerList
jteresco May 21, 2020
655bd75
2 region-splitting bugfixes
yakra May 24, 2020
6a67a7e
Merge pull request #331 from yakra/TravelerList
jteresco May 24, 2020
09e6864
trim whitespace from datacheckfps.csv lines
yakra May 27, 2020
d174961
Merge pull request #332 from yakra/TravelerList
jteresco May 30, 2020
73ed9a3
Feature: Multi-region list file entries
yakra May 31, 2020
c15f3c2
mutex fixes
yakra Jun 1, 2020
feabf2f
Merge pull request #333 from yakra/TravelerList
jteresco Jun 1, 2020
e40e80b
LABEL_INVALID_CHAR fix
yakra Jun 2, 2020
a9b4b39
Merge pull request #336 from yakra/datacheck
jteresco Jun 3, 2020
166286b
ConnectedRoute checks & fixes
yakra Jun 11, 2020
b2b7e13
DISCONNECTED_ROUTE comment & always_error entry
yakra Jun 11, 2020
a6c75d3
Merge pull request #339 from yakra/datacheck
jteresco Jun 11, 2020
96465fb
BUS_WITH_I datacheck improvements
yakra Jun 12, 2020
cdcfbb5
detect BUS_WITH_I for I-##C (e.g. I-69C)
yakra Jun 12, 2020
ab529b8
Merge pull request #340 from yakra/datacheck
jteresco Jun 12, 2020
0e34dfa
invalid char datacheck improvements
yakra Jun 12, 2020
3fc61aa
Merge pull request #342 from yakra/datacheck
jteresco Jun 13, 2020
ec822f0
LABEL_INVALID_CHAR for all points
yakra Jun 14, 2020
36affb6
Merge pull request #343 from yakra/datacheck
jteresco Jun 14, 2020
c039a97
LACKS_GENERIC datacheck
yakra Jun 15, 2020
c878b32
DUPLICATE_LABEL userlog warning bugfix
yakra Jun 15, 2020
0dc0eb9
BUS_WITH_I: *closed points
yakra Jun 16, 2020
28d3ec3
Merge pull request #344 from yakra/datacheck
jteresco Jun 16, 2020
d30d3ba
http->https in message
jteresco Jul 1, 2020
35e580f
Merge branch 'master' of https://github.com/TravelMapping/DataProcessing
jteresco Jul 1, 2020
15c53f4
remove extraneous condition
yakra Aug 7, 2020
11be967
Merge pull request #348 from yakra/datacheck
jteresco Aug 13, 2020
e7eb26e
INTERSTATE_NO_HYPHEN
yakra Nov 1, 2020
2ea7f5f
Merge pull request #350 from yakra/datacheck
jteresco Nov 2, 2020
76f9e35
prevent empty routes from crashing DISCONNECTED_ROUTE datacheck
yakra Nov 13, 2020
3d56bd9
reduce Python<->C++ siteupdate.log diffs
yakra Nov 13, 2020
5dbed5a
Merge pull request #352 from yakra/datacheck
jteresco Nov 13, 2020
53ca19d
con_total_miles cleanup
yakra Nov 15, 2020
bb1a391
C++ nmp_merged progress indication
yakra Nov 15, 2020
d27abbf
consolidate USA-only datachecks
yakra Nov 15, 2020
a7a9efd
US_LETTER datacheck
yakra Nov 15, 2020
05faea9
detect INTERSTATE_NO_HYPHEN in closed waypoints
yakra Nov 15, 2020
74b7aea
delete vestigial LABEL_SELFREF comments
yakra Nov 16, 2020
9eec0ed
Merge pull request #357 from yakra/datacheck
jteresco Nov 16, 2020
ad330ff
Merge pull request #354 from yakra/CodeOrg
jteresco Nov 16, 2020
dc37d8a
expand US_LETTER datacheck
yakra Nov 17, 2020
8da52b5
Merge pull request #359 from yakra/datacheck
jteresco Nov 19, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 18 additions & 9 deletions siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.cpp
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
ConnectedRoute::ConnectedRoute(std::string &line, HighwaySystem *sys, ErrorList &el, std::list<Route> &route_list)
ConnectedRoute::ConnectedRoute(std::string &line, HighwaySystem *sys, ErrorList &el)
{ mileage = 0;

// parse chopped routes csv line
Expand DownExpand Up@@ -29,19 +29,28 @@ ConnectedRoute::ConnectedRoute(std::string &line, HighwaySystem *sys, ErrorList
el.add_error("groupname > " + std::to_string(DBFieldLength::city)
+ " bytes in " + system->systemname + "_con.csv line: " + line);
// roots
lower(roots_str.data());
int rootOrder = 0;
size_t l = 0;
for (size_t r = 0; r != -1; l = r+1)
{ r = roots_str.find(',', l);
Route *root = route_by_root(roots_str.substr(l, r-l), route_list);
if (!root) el.add_error("Could not find Route matching ConnectedRoute root " + roots_str.substr(l, r-l) +
" in system " + system->systemname + '.');
else { roots.push_back(root);
try { Route *root = Route::root_hash.at(roots_str.substr(l, r-l));
roots.push_back(root);
if (root->con_route)
el.add_error("Duplicate root in " + sys->systemname + "_con.csv: " + root->root +
" already in " + root->con_route->system->systemname + "_con.csv");
if (system != root->system)
el.add_error("System mismatch: chopped route " + root->root + " from " + root->system->systemname +
".csv in connected route in " + system->systemname + "_con.csv");
root->con_route = this;
// save order of route in connected route
root->rootOrder = rootOrder;
}
rootOrder++;
rootOrder++;
}
catch (std::out_of_range& oor)
{ el.add_error("Could not find Route matching ConnectedRoute root " + roots_str.substr(l, r-l) +
" in system " + system->systemname + '.');
}
}
if (roots.size() < 1) el.add_error("No roots in " + system->systemname + "_con.csv line: " + line);
}
Expand DownExpand Up@@ -72,7 +81,7 @@ std::string ConnectedRoute::readable_name()
return ans;
}

std::string ConnectedRoute::list_lines(int pos, int len, std::string newline, size_t indent)
/*std::string ConnectedRoute::list_lines(int pos, int len, std::string newline, size_t indent)
{ // return .list file lines marking (len) consecutive
// segments, starting at waypoint (pos) segments into route
//std::cout << "\nDEBUG: list_lines for " << readable_name() << " (" << roots.size() << " connected root(s))" << std::endl;
Expand All@@ -86,4 +95,4 @@ std::string ConnectedRoute::list_lines(int pos, int len, std::string newline, si
// strip final newline
while (lines.back() == '\n' || lines.back() == '\r') lines.pop_back();
return lines;
}
}//*/
4 changes: 2 additions & 2 deletions siteupdate/cplusplus/classes/ConnectedRoute/ConnectedRoute.h
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,10 +12,10 @@ class ConnectedRoute

double mileage; // will be computed for routes in active & preview systems

ConnectedRoute(std::string &, HighwaySystem *, ErrorList &, std::list<Route> &);
ConnectedRoute(std::string &, HighwaySystem *, ErrorList &);

std::string connected_rtes_line();
std::string csv_line();
std::string readable_name();
std::string list_lines(int, int, std::string, size_t);
//std::string list_lines(int, int, std::string, size_t);
};
10 changes: 6 additions & 4 deletions siteupdate/cplusplus/classes/DatacheckEntry.cpp
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,19 +3,21 @@ class DatacheckEntry

route is a pointer to the route with a datacheck error

labels is a list of labels that are related to the error (such
as the endpoints of a too-long segment or the three points that
form a sharp angle)
label1, label2 & label3 are labels that are related to the error
(such as the endpoints of a too-long segment or the three points
that form a sharp angle)

code is the error code | info is additional
string, one of: | information, if used:
-----------------------+--------------------------------------------
BAD_ANGLE |
BUS_WITH_I |
DISCONNECTED_ROUTE | adjacent root's expected connection point
DUPLICATE_COORDS | coordinate pair
DUPLICATE_LABEL |
HIDDEN_JUNCTION | number of incident edges in TM master graph
HIDDEN_TERMINUS |
INTERSTATE_NO_HYPHEN |
INVALID_FINAL_CHAR | final character in label
INVALID_FIRST_CHAR | first character in label other than *
LABEL_INVALID_CHAR |
Expand All@@ -34,7 +36,7 @@ class DatacheckEntry
NONTERMINAL_UNDERSCORE |
OUT_OF_BOUNDS | coordinate pair
SHARP_ANGLE | angle in degrees
US_BANNER |
US_LETTER |
VISIBLE_DISTANCE | distance in miles
VISIBLE_HIDDEN_COLOC | hidden point at same coordinates

Expand Down
5 changes: 0 additions & 5 deletions siteupdate/cplusplus/classes/GraphGeneration/HGEdge.cpp
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,3 @@
// constants for more human-readable format masks
const unsigned char HGEdge::simple = 1;
const unsigned char HGEdge::collapsed = 2;
const unsigned char HGEdge::traveled = 4;

HGEdge::HGEdge(HighwaySegment *s, HighwayGraph *graph)
{ // initial construction is based on a HighwaySegment
s_written = 0; // simple
Expand Down
6 changes: 3 additions & 3 deletions siteupdate/cplusplus/classes/GraphGeneration/HGEdge.h
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,9 +12,9 @@ class HGEdge
unsigned char format;

// constants for more human-readable format masks
static const unsigned char simple;
static const unsigned char collapsed;
static const unsigned char traveled;
static const unsigned char simple = 1;
static const unsigned char collapsed = 2;
static const unsigned char traveled = 4;

HGEdge(HighwaySegment *, HighwayGraph *);
HGEdge(HGVertex *, unsigned char);
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -96,7 +96,7 @@ class HighwayGraph
// compress edges adjacent to hidden vertices
counter = 0;
std::cout << et.et() + "Compressing collapsed edges" << std::flush;
for (std::pair<const Waypoint*, HGVertex*> wv : vertices)
for (std::pair<Waypoint* const, HGVertex*>& wv : vertices)
{ if (counter % 10000 == 0) std::cout << '.' << std::flush;
counter++;
if (!wv.second->visibility)
Expand DownExpand Up@@ -146,7 +146,7 @@ class HighwayGraph
} // end ctor

void clear()
{ for (std::pair<const Waypoint*, HGVertex*> wv : vertices) delete wv.second;
{ for (std::pair<Waypoint* const, HGVertex*>& wv : vertices) delete wv.second;
vertex_names.clear();
waypoint_naming_log.clear();
vertices.clear();
Expand Down
16 changes: 8 additions & 8 deletions siteupdate/cplusplus/classes/GraphGeneration/PlaceRadius.cpp
Original file line numberDiff line numberDiff line change
Expand Up@@ -10,10 +10,10 @@ bool PlaceRadius::contains_vertex(HGVertex *v) {return contains_vertex(v->lat, v
bool PlaceRadius::contains_vertex(double vlat, double vlng)
{ /* return whether coordinates are within this area */
// convert to radians to compute distance
double rlat1 = lat * (Waypoint::pi/180);
double rlng1 = lng * (Waypoint::pi/180);
double rlat2 = vlat * (Waypoint::pi/180);
double rlng2 = vlng * (Waypoint::pi/180);
double rlat1 = lat * (pi/180);
double rlng1 = lng * (pi/180);
double rlat2 = vlat * (pi/180);
double rlng2 = vlng * (pi/180);

/* original formula
double ans = acos(cos(rlat1)*cos(rlng1)*cos(rlat2)*cos(rlng2) +\
Expand DownExpand Up@@ -48,10 +48,10 @@ std::unordered_set<HGVertex*> PlaceRadius::vertices(WaypointQuadtree *qt, Highwa

// N/S sanity check: If lat is <= r/2 miles to the N or S pole, lngdelta calculation will fail.
// In these cases, our place radius will span the entire "width" of the world, from -180 to +180 degrees.
if (90-fabs(lat)*(Waypoint::pi/180) <= r/7926.2) return v_search(qt, g, -180, +180);
if (90-fabs(lat)*(pi/180) <= r/7926.2) return v_search(qt, g, -180, +180);

// width, in degrees longitude, of our bounding box for quadtree search
double lngdelta = acos((cos(r/3963.1) - pow(sin(lat*(Waypoint::pi/180)),2)) / pow(cos(lat*(Waypoint::pi/180)),2)) / (Waypoint::pi/180);
double lngdelta = acos((cos(r/3963.1) - pow(sin(lat*(pi/180)),2)) / pow(cos(lat*(pi/180)),2)) / (pi/180);
double w_bound = lng-lngdelta;
double e_bound = lng+lngdelta;

Expand DownExpand Up@@ -92,8 +92,8 @@ std::unordered_set<HGVertex*> PlaceRadius::v_search(WaypointQuadtree *qt, Highwa
// if we're not a terminal quadrant, we need to determine which
// of our child quadrants we need to search and recurse into each
else { //printf("DEBUG: recursive case, mid_lat=%.17g mid_lng=%.17g\n", qt->mid_lat, qt->mid_lng); fflush(stdout);
bool look_n = (lat + r/3963.1/(Waypoint::pi/180)) >= qt->mid_lat;
bool look_s = (lat - r/3963.1/(Waypoint::pi/180)) <= qt->mid_lat;
bool look_n = (lat + r/3963.1/(pi/180)) >= qt->mid_lat;
bool look_s = (lat - r/3963.1/(pi/180)) <= qt->mid_lat;
bool look_e = e_bound >= qt->mid_lng;
bool look_w = w_bound <= qt->mid_lng;
//std::cout << "DEBUG: recursive case, " << look_n << " " << look_s << " " << look_e << " " << look_w << std::endl;
Expand Down
16 changes: 6 additions & 10 deletions siteupdate/cplusplus/classes/HighwaySegment/HighwaySegment.cpp
Original file line numberDiff line numberDiff line change
Expand Up@@ -49,19 +49,15 @@ unsigned int HighwaySegment::index()
/*std::string HighwaySegment::concurrent_travelers_sanity_check()
{ if (route->system->devel()) return "";
if (concurrent)
for (HighwaySegment *conc : *concurrent)
{ if (clinched_by.size() != conc->clinched_by.size())
{ if (conc->route->system->devel()) continue;
for (HighwaySegment *other : *concurrent)
{ if (clinched_by.size() != other->clinched_by.size())
{ if (other->route->system->devel()) continue;
return "[" + str() + "] clinched by " + std::to_string(clinched_by.size()) + " travelers; [" \
+ conc->str() + "] clinched by " + std::to_string(conc->clinched_by.size()) + '\n';
+ other->str() + "] clinched by " + std::to_string(other->clinched_by.size()) + '\n';
}
else for (TravelerList *t : clinched_by)
{ std::list<TravelerList*>::iterator ct;
for (ct = conc->clinched_by.begin(); ct != conc->clinched_by.end(); ct++)
if (*ct == t) break;
if (ct == conc->clinched_by.end())
return t->traveler_name + " has clinched [" + str() + "], but not [" + conc->str() + "]\n";
}
if (other->clinched_by.find(t) == other->clinched_by.end())
return t->traveler_name + " has clinched [" + str() + "], but not [" + other->str() + "]\n";
}
return "";
}//*/
Expand Down
15 changes: 10 additions & 5 deletions siteupdate/cplusplus/classes/HighwaySystem.cpp
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
//FIXME try to break strtok. That goes for all strtok project-wide.
class HighwaySystem
{ /* This class encapsulates the contents of one .csv file
that represents the collection of highways within a system.
Expand DownExpand Up@@ -28,6 +27,8 @@ class HighwaySystem
std::list<ConnectedRoute> con_route_list;
std::unordered_map<Region*, double> mileage_by_region;
std::unordered_set<HGVertex*> vertices;
std::unordered_set<std::string>listnamesinuse, unusedaltroutenames;
std::mutex lniu_mtx, uarn_mtx;
bool is_valid;

HighwaySystem(std::string &line, ErrorList &el, std::string path, std::string &systemsfile,
Expand DownExpand Up@@ -74,12 +75,15 @@ class HighwaySystem
if (level_str != "active" && level_str != "preview" && level_str != "devel")
el.add_error("Unrecognized level in " + systemsfile + " line: " + line);

std::cout << systemname << '.' << std::flush;

// read chopped routes CSV
file.open(path+"/"+systemname+".csv");
if (!file) el.add_error("Could not open "+path+"/"+systemname+".csv");
else { getline(file, line); // ignore header line
while(getline(file, line))
{ if (line.back() == 0x0D) line.erase(line.end()-1); // trim DOS newlines
{ // trim DOS newlines & trailing whitespace
while ( strchr("\r\t ", line.back()) ) line.pop_back();
if (line.empty()) continue;
route_list.emplace_back(line, this, el, region_hash);
if (route_list.back().root.empty())
Expand All@@ -95,9 +99,10 @@ class HighwaySystem
if (!file) el.add_error("Could not open "+path+"/"+systemname+"_con.csv");
else { getline(file, line); // ignore header line
while(getline(file, line))
{ if (line.back() == 0x0D) line.erase(line.end()-1); // trim DOS newlines
{ // trim DOS newlines & trailing whitespace
while ( strchr("\r\t ", line.back()) ) line.pop_back();
if (line.empty()) continue;
con_route_list.emplace_back(line, this, el, route_list);
con_route_list.emplace_back(line, this, el);
}
}
file.close();
Expand DownExpand Up@@ -126,7 +131,7 @@ class HighwaySystem
/* Return total system mileage across all regions */
double total_mileage()
{ double mi = 0;
for (std::pair<Region*, double> rm : mileage_by_region) mi += rm.second;
for (std::pair<Region* const, double>& rm : mileage_by_region) mi += rm.second;
return mi;
}

Expand Down
4 changes: 2 additions & 2 deletions siteupdate/cplusplus/classes/Region.cpp
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
std::pair<std::string, std::string> *country_or_continent_by_code(std::string code, std::vector<std::pair<std::string, std::string>> &pair_vector)
{ for (std::vector<std::pair<std::string, std::string>>::iterator c = pair_vector.begin(); c != pair_vector.end(); c++)
if (c->first == code) return &*c;
{ for (std::pair<std::string, std::string>& c : pair_vector)
if (c.first == code) return &c;
return 0;
}

Expand Down
62 changes: 51 additions & 11 deletions siteupdate/cplusplus/classes/Route/Route.cpp
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
std::unordered_map<std::string, Route*> Route::root_hash, Route::pri_list_hash, Route::alt_list_hash;
std::mutex Route::awf_mtx;

Route::Route(std::string &line, HighwaySystem *sys, ErrorList &el, std::unordered_map<std::string, Region*> &region_hash)
Expand All@@ -7,6 +8,7 @@ Route::Route(std::string &line, HighwaySystem *sys, ErrorList &el, std::unordere
mileage = 0;
rootOrder = -1; // order within connected route
region = 0; // if this stays 0, setup has failed due to bad .csv data
is_reversed = 0;

// parse chopped routes csv line
size_t NumFields = 8;
Expand DownExpand Up@@ -52,11 +54,38 @@ Route::Route(std::string &line, HighwaySystem *sys, ErrorList &el, std::unordere
if (root.size() > DBFieldLength::root)
el.add_error("Root > " + std::to_string(DBFieldLength::root)
+ " bytes in " + system->systemname + ".csv line: " + line);
lower(root.data());
// alt_route_names
size_t l = 0;
for (size_t r = 0; r != -1; l = r+1)
{ r = arn_str.find(',', l);
alt_route_names.emplace_back(arn_str, l, r-l);
upper(arn_str.data());
size_t len;
for (size_t pos = 0; pos < arn_str.size(); pos += len+1)
{ len = strcspn(arn_str.data()+pos, ",");
alt_route_names.emplace_back(arn_str, pos, len);
}

// insert into root_hash, checking for duplicate root entries
if (!root_hash.insert(std::pair<std::string, Route*>(root, this)).second)
el.add_error("Duplicate root in " + system->systemname + ".csv: " + root +
" already in " + root_hash.at(root)->system->systemname + ".csv");
// insert list name into pri_list_hash, checking for duplicate .list names
std::string list_name = upper(readable_name());
if (alt_list_hash.find(list_name) != alt_list_hash.end())
el.add_error("Duplicate main list name in " + root + ": '" + readable_name() +
"' already points to " + alt_list_hash.at(list_name)->root);
else if (!pri_list_hash.insert(std::pair<std::string,Route*>(list_name, this)).second)
el.add_error("Duplicate main list name in " + root + ": '" + readable_name() +
"' already points to " + pri_list_hash.at(list_name)->root);
// insert alt names into alt_list_hash, checking for duplicate .list names
for (std::string& a : alt_route_names)
{ list_name = upper(rg_str + ' ' + a);
if (pri_list_hash.find(list_name) != pri_list_hash.end())
el.add_error("Duplicate alt route name in " + root + ": '" + region->code + ' ' + a +
"' already points to " + pri_list_hash.at(list_name)->root);
else if (!alt_list_hash.insert(std::pair<std::string, Route*>(list_name, this)).second)
el.add_error("Duplicate alt route name in " + root + ": '" + region->code + ' ' + a +
"' already points to " + alt_list_hash.at(list_name)->root);
// populate unused set
system->unusedaltroutenames.insert(list_name);
}
}

Expand DownExpand Up@@ -132,16 +161,16 @@ double Route::clinched_by_traveler(TravelerList *t)
return miles;
}

std::string Route::list_line(int beg, int end)
/*std::string Route::list_line(int beg, int end)
{ /* Return a .list file line from (beg) to (end),
these being indices to the point_list vector.
These values can be "out-of-bounds" when getting lines
for connected routes. If so, truncate or return "". */
if (beg >= int(point_list.size()) || end <= 0) return "";
/* if (beg >= int(point_list.size()) || end <= 0) return "";
if (end >= int(point_list.size())) end = point_list.size()-1;
if (beg < 0) beg = 0;
return readable_name() + " " + point_list[beg]->label + " " + point_list[end]->label;
}
}//*/

void Route::write_nmp_merged(std::string filename)
{ mkdir(filename.data(), 0777);
Expand DownExpand Up@@ -180,8 +209,19 @@ void Route::write_nmp_merged(std::string filename)
wptfile.close();
}

Route *route_by_root(std::string root, std::list<Route> &route_list)
{ for (std::list<Route>::iterator r = route_list.begin(); r != route_list.end(); r++)
if (r->root == root) return &*r;
return 0;
inline void Route::store_traveled_segments(TravelerList* t, unsigned int beg, unsigned int end)
{ // store clinched segments with traveler and traveler with segments
for (unsigned int pos = beg; pos < end; pos++)
{ HighwaySegment *hs = segment_list[pos];
hs->add_clinched_by(t);
t->clinched_segments.insert(hs);
}
}

inline Waypoint* Route::con_beg()
{ return is_reversed ? point_list.back() : point_list.front();
}

inline Waypoint* Route::con_end()
{ return is_reversed ? point_list.front() : point_list.back();
}
Loading