Skip to content

subgraph parallelism; thread safety fixes#231

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jteresco merged 2 commits into from
Jun 24, 2019
Merged

subgraph parallelism; thread safety fixes #231

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4879245
improve subgraph generation parallelism
yakra Jun 20, 2019
835bd72
threads: "modify lists" -> "modify iterators"
yakra Jun 23, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -491,7 +491,8 @@ class HighwayGraph
// restricted by regions in the list if given,
// by systems in the list if given,
// or to within a given area if placeradius is given
void write_subgraphs_tmg(std::vector<GraphListEntry> &graph_vector, std::string path, size_t graphnum, unsigned int threadnum, WaypointQuadtree *qt)
void write_subgraphs_tmg( std::vector<GraphListEntry> &graph_vector, std::string path, size_t graphnum,
unsigned int threadnum, WaypointQuadtree *qt, ElapsedTime *et)
{ unsigned int cv_count, tv_count;
std::ofstream simplefile((path+graph_vector[graphnum].filename()).data());
std::ofstream collapfile((path+graph_vector[graphnum+1].filename()).data());
Expand All@@ -508,6 +509,10 @@ class HighwayGraph
{ t->traveler_num[threadnum] = travnum;
travnum++;
}
#ifdef threading_enabled
if (graph_vector[graphnum].cat != graph_vector[graphnum-1].cat)
std::cout << '\n' + et->et() + "Writing " + graph_vector[graphnum].category() + " graphs.\n";
#endif
std::cout << graph_vector[graphnum].tag()
<< '(' << mv.size() << ',' << mse.size() << ") "
<< '(' << cv_count << ',' << mce.size() << ") "
Expand Down
329 changes: 10 additions & 319 deletions siteupdate/cplusplus/functions/graph_generation.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,336 +53,27 @@ else { list<Region*> *regions;

graph_types.push_back({"master", "All Travel Mapping Data", "These graphs contain all routes currently plotted in the Travel Mapping project."});
size_t graphnum = 3;


// graphs restricted by place/area - from areagraphs.csv file
cout << '\n' << et.et() << "Creating area data graphs." << endl;
filename = args.highwaydatapath+"/graphs/areagraphs.csv";
file.open(filename.data());
getline(file, line); // ignore header line
list<PlaceRadius> area_list;
while (getline(file, line))
{ vector<char*> fields;
char *cline = new char[line.size()+1];
strcpy(cline, line.data());
for (char *token = strtok(cline, ";"); token; token = strtok(0, ";")) fields.push_back(token);
if (fields.size() != 5)
{ cout << "Could not parse areagraphs.csv line: " << cline << endl;
delete[] cline;
continue;
}
area_list.emplace_back(fields[0], fields[1], fields[2], fields[3], fields[4]);
delete[] cline;
}
file.close();

// add entries to graph_vector
for (PlaceRadius &a : area_list)
{ graph_vector.emplace_back(a.base + to_string(a.r) + "-area", a.place + " (" + to_string(a.r) + " mi radius)",
's', 'a', (list<Region*>*)0, (list<HighwaySystem*>*)0, &a);
graph_vector.emplace_back(a.base + to_string(a.r) + "-area", a.place + " (" + to_string(a.r) + " mi radius)",
'c', 'a', (list<Region*>*)0, (list<HighwaySystem*>*)0, &a);
graph_vector.emplace_back(a.base + to_string(a.r) + "-area", a.place + " (" + to_string(a.r) + " mi radius)",
't', 'a', (list<Region*>*)0, (list<HighwaySystem*>*)0, &a);
}
// write new graph_vector entries to disk
#ifdef threading_enabled
// set up for threaded subgraph generation
for (unsigned int t = 0; t < args.numthreads; t++)
thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &all_waypoints);
for (unsigned int t = 0; t < args.numthreads; t++)
thr[t]->join();
for (unsigned int t = 0; t < args.numthreads; t++)
delete thr[t];
#else
while (graphnum < graph_vector.size())
{ graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &all_waypoints);
graphnum += 3;
}
cout << et.et() << "Setting up subgraphs." << flush;
#endif
graph_types.push_back({"area", "Routes Within a Given Radius of a Place",
"These graphs contain all routes currently plotted within the given distance radius of the given place."});
cout << '!' << endl;
area_list.clear();


// Graphs restricted by region
cout << et.et() << "Creating regional data graphs." << endl;

// We will create graph data and a graph file for each region that includes
// any active or preview systems

// add entries to graph_vector
for (Region &region : all_regions)
{ if (region.active_preview_mileage == 0) continue;
regions = new list<Region*>(1, &region);
// deleted on termination of program
graph_vector.emplace_back(region.code + "-region", region.name + " (" + region.type + ")",
's', 'r', regions, (list<HighwaySystem*>*)0, (PlaceRadius*)0);
graph_vector.emplace_back(region.code + "-region", region.name + " (" + region.type + ")",
'c', 'r', (list<Region*>*)0, (list<HighwaySystem*>*)0, (PlaceRadius*)0);
graph_vector.emplace_back(region.code + "-region", region.name + " (" + region.type + ")",
't', 'r', (list<Region*>*)0, (list<HighwaySystem*>*)0, (PlaceRadius*)0);
}
// write new graph_vector entries to disk
#include "subgraphs/continent.cpp"
#include "subgraphs/multisystem.cpp"
#include "subgraphs/system.cpp"
#include "subgraphs/country.cpp"
#include "subgraphs/multiregion.cpp"
#include "subgraphs/area.cpp"
#include "subgraphs/region.cpp"
#ifdef threading_enabled
// set up for threaded subgraph generation
for (unsigned int t = 0; t < args.numthreads; t++)
thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &all_waypoints);
for (unsigned int t = 0; t < args.numthreads; t++)
thr[t]->join();
for (unsigned int t = 0; t < args.numthreads; t++)
delete thr[t];
#else
while (graphnum < graph_vector.size())
{ graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &all_waypoints);
graphnum += 3;
}
#endif
graph_types.push_back({"region", "Routes Within a Single Region", "These graphs contain all routes currently plotted within the given region."});
cout << "!" << endl;


// Graphs restricted by system - from systemgraphs.csv file
cout << et.et() << "Creating system data graphs." << endl;

// We will create graph data and a graph file for only a few interesting
// systems, as many are not useful on their own
HighwaySystem *h;
filename = args.highwaydatapath+"/graphs/systemgraphs.csv";
file.open(filename.data());
getline(file, line); // ignore header line

// add entries to graph_vector
while (getline(file, line))
{ h = 0;
for (HighwaySystem *hs : highway_systems)
if (hs->systemname == line)
{ h = hs;
break;
}
if (h)
{ systems = new list<HighwaySystem*>(1, h);
// deleted on termination of program
graph_vector.emplace_back(h->systemname + "-system", h->systemname + " (" + h->fullname + ")",
's', 's', (list<Region*>*)0, systems, (PlaceRadius*)0);
graph_vector.emplace_back(h->systemname + "-system", h->systemname + " (" + h->fullname + ")",
'c', 's', (list<Region*>*)0, (list<HighwaySystem*>*)0, (PlaceRadius*)0);
graph_vector.emplace_back(h->systemname + "-system", h->systemname + " (" + h->fullname + ")",
't', 's', (list<Region*>*)0, (list<HighwaySystem*>*)0, (PlaceRadius*)0);
}
}
file.close();
// write new graph_vector entries to disk
#ifdef threading_enabled
// set up for threaded subgraph generation
for (unsigned int t = 0; t < args.numthreads; t++)
thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &all_waypoints);
thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &all_waypoints, &et);
for (unsigned int t = 0; t < args.numthreads; t++)
thr[t]->join();
for (unsigned int t = 0; t < args.numthreads; t++)
delete thr[t];
#else
while (graphnum < graph_vector.size())
{ graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &all_waypoints);
graphnum += 3;
}
#endif
if (h) graph_types.push_back({"system", "Routes Within a Single Highway System",
"These graphs contain the routes within a single highway system and are not restricted by region."});
cout << "!" << endl;


// Some additional interesting graphs, the "multisystem" graphs
cout << et.et() << "Creating multisystem graphs." << endl;

filename = args.highwaydatapath+"/graphs/multisystem.csv";
file.open(filename.data());
getline(file, line); // ignore header line

// add entries to graph_vector
while (getline(file, line))
{ vector<char*> fields;
char *cline = new char[line.size()+1];
strcpy(cline, line.data());
for (char *token = strtok(cline, ";"); token; token = strtok(0, ";")) fields.push_back(token);
if (fields.size() != 3)
{ cout << "Could not parse multisystem.csv line: " << line << endl;
delete[] cline;
continue;
}
systems = new list<HighwaySystem*>;
// deleted on termination of program
for(char* s = strtok(fields[2], ","); s; s = strtok(0, ","))
for (HighwaySystem *h : highway_systems)
if (s == h->systemname)
{ systems->push_back(h);
break;
}
graph_vector.emplace_back(fields[1], fields[0],
's', 'S', (list<Region*>*)0, systems, (PlaceRadius*)0);
graph_vector.emplace_back(fields[1], fields[0],
'c', 'S', (list<Region*>*)0, (list<HighwaySystem*>*)0, (PlaceRadius*)0);
graph_vector.emplace_back(fields[1], fields[0],
't', 'S', (list<Region*>*)0, (list<HighwaySystem*>*)0, (PlaceRadius*)0);
delete[] cline;
}
file.close();
// write new graph_vector entries to disk
#ifdef threading_enabled
// set up for threaded subgraph generation
for (unsigned int t = 0; t < args.numthreads; t++)
thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &all_waypoints);
for (unsigned int t = 0; t < args.numthreads; t++)
thr[t]->join();
for (unsigned int t = 0; t < args.numthreads; t++)
delete thr[t];
#else
while (graphnum < graph_vector.size())
{ graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &all_waypoints);
graphnum += 3;
}
#endif
graph_types.push_back({"multisystem", "Routes Within Multiple Highway Systems", "These graphs contain the routes within a set of highway systems."});
cout << "!" << endl;


// Some additional interesting graphs, the "multiregion" graphs
cout << et.et() << "Creating multiregion graphs." << endl;

filename = args.highwaydatapath+"/graphs/multiregion.csv";
file.open(filename.data());
getline(file, line); // ignore header line

// add entries to graph_vector
while (getline(file, line))
{ vector<char*> fields;
char *cline = new char[line.size()+1];
strcpy(cline, line.data());
for (char *token = strtok(cline, ";"); token; token = strtok(0, ";")) fields.push_back(token);
if (fields.size() != 3)
{ cout << "Could not parse multiregion.csv line: " << line << endl;
delete[] cline;
continue;
}
regions = new list<Region*>;
// deleted on termination of program
for(char* rg = strtok(fields[2], ","); rg; rg = strtok(0, ","))
for (Region &r : all_regions)
if (rg == r.code)
{ regions->push_back(&r);
break;
}
graph_vector.emplace_back(fields[1], fields[0],
's', 'R', regions, (list<HighwaySystem*>*)0, (PlaceRadius*)0);
graph_vector.emplace_back(fields[1], fields[0],
'c', 'R', (list<Region*>*)0, (list<HighwaySystem*>*)0, (PlaceRadius*)0);
graph_vector.emplace_back(fields[1], fields[0],
't', 'R', (list<Region*>*)0, (list<HighwaySystem*>*)0, (PlaceRadius*)0);
delete[] cline;
}
file.close();
// write new graph_vector entries to disk
#ifdef threading_enabled
// set up for threaded subgraph generation
for (unsigned int t = 0; t < args.numthreads; t++)
thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &all_waypoints);
for (unsigned int t = 0; t < args.numthreads; t++)
thr[t]->join();
for (unsigned int t = 0; t < args.numthreads; t++)
delete thr[t];
#else
while (graphnum < graph_vector.size())
{ graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &all_waypoints);
graphnum += 3;
}
#endif
graph_types.push_back({"multiregion", "Routes Within Multiple Regions", "These graphs contain the routes within a set of regions."});
cout << "!" << endl;


// country graphs - we find countries that have regions
// that have routes with active or preview mileage
cout << et.et() << "Creating country graphs." << endl;

// add entries to graph_vector
for (pair<string, string> &c : countries)
{ regions = new list<Region*>;
// deleted on termination of program
for (Region &r : all_regions)
// does it match this country and have routes?
if (&c == r.country && r.active_preview_mileage)
regions->push_back(&r);
// does it have at least two? if none, no data,
// if 1 we already generated a graph for that one region
if (regions->size() < 2) delete regions;
else { graph_vector.emplace_back(c.first + "-country", c.second + " All Routes in Country",
's', 'c', regions, (list<HighwaySystem*>*)0, (PlaceRadius*)0);
graph_vector.emplace_back(c.first + "-country", c.second + " All Routes in Country",
'c', 'c', (list<Region*>*)0, (list<HighwaySystem*>*)0, (PlaceRadius*)0);
graph_vector.emplace_back(c.first + "-country", c.second + " All Routes in Country",
't', 't', (list<Region*>*)0, (list<HighwaySystem*>*)0, (PlaceRadius*)0);
}
}
// write new graph_vector entries to disk
#ifdef threading_enabled
// set up for threaded subgraph generation
for (unsigned int t = 0; t < args.numthreads; t++)
thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &all_waypoints);
for (unsigned int t = 0; t < args.numthreads; t++)
thr[t]->join();
for (unsigned int t = 0; t < args.numthreads; t++)
delete thr[t];
#else
while (graphnum < graph_vector.size())
{ graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &all_waypoints);
graphnum += 3;
}
#endif
graph_types.push_back({"country", "Routes Within a Single Multi-Region Country",
"These graphs contain the routes within a single country that is composed of multiple regions that contain plotted routes. " +
string("Countries consisting of a single region are represented by their regional graph.")});
cout << "!" << endl;


// continent graphs -- any continent with data will be created
cout << et.et() << "Creating continent graphs." << endl;

// add entries to graph_vector
for (pair<string, string> &c : continents)
{ regions = new list<Region*>;
// deleted on termination of program
for (Region &r : all_regions)
// does it match this continent and have routes?
if (&c == r.continent && r.active_preview_mileage)
regions->push_back(&r);
// generate for any continent with at least 1 region with mileage
if (regions->size() < 1) delete regions;
else { graph_vector.emplace_back(c.first + "-continent", c.second + " All Routes on Continent",
's', 'C', regions, (list<HighwaySystem*>*)0, (PlaceRadius*)0);
graph_vector.emplace_back(c.first + "-continent", c.second + " All Routes on Continent",
'c', 'C', (list<Region*>*)0, (list<HighwaySystem*>*)0, (PlaceRadius*)0);
graph_vector.emplace_back(c.first + "-continent", c.second + " All Routes on Continent",
't', 'C', (list<Region*>*)0, (list<HighwaySystem*>*)0, (PlaceRadius*)0);
}
}
// write new graph_vector entries to disk
#ifdef threading_enabled
// set up for threaded subgraph generation
for (unsigned int t = 0; t < args.numthreads; t++)
thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &all_waypoints);
for (unsigned int t = 0; t < args.numthreads; t++)
thr[t]->join();
for (unsigned int t = 0; t < args.numthreads; t++)
delete thr[t];
#else
while (graphnum < graph_vector.size())
{ graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &all_waypoints);
graphnum += 3;
}
cout << '!' << endl;
#endif
graph_types.push_back({"continent", "Routes Within a Continent", "These graphs contain the routes on a continent."});
cout << "!" << endl;
}

graph_data.clear();
Loading