From 4879245fdfa727e57f90cd33b110a7ddd14e9bd6 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Thu, 20 Jun 2019 12:03:10 -0400 Subject: [PATCH 1/2] improve subgraph generation parallelism --- .../classes/GraphGeneration/HighwayGraph.cpp | 7 +- .../cplusplus/functions/graph_generation.cpp | 329 +----------------- .../cplusplus/functions/subgraphs/area.cpp | 42 +++ .../functions/subgraphs/continent.cpp | 31 ++ .../cplusplus/functions/subgraphs/country.cpp | 35 ++ .../functions/subgraphs/multiregion.cpp | 45 +++ .../functions/subgraphs/multisystem.cpp | 45 +++ .../cplusplus/functions/subgraphs/region.cpp | 28 ++ .../cplusplus/functions/subgraphs/system.cpp | 41 +++ .../cplusplus/threads/SubgraphThread.cpp | 4 +- 10 files changed, 285 insertions(+), 322 deletions(-) create mode 100644 siteupdate/cplusplus/functions/subgraphs/area.cpp create mode 100644 siteupdate/cplusplus/functions/subgraphs/continent.cpp create mode 100644 siteupdate/cplusplus/functions/subgraphs/country.cpp create mode 100644 siteupdate/cplusplus/functions/subgraphs/multiregion.cpp create mode 100644 siteupdate/cplusplus/functions/subgraphs/multisystem.cpp create mode 100644 siteupdate/cplusplus/functions/subgraphs/region.cpp create mode 100644 siteupdate/cplusplus/functions/subgraphs/system.cpp diff --git a/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp b/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp index ad0a8940..7bed22e8 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp +++ b/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp @@ -491,7 +491,8 @@ class HighwayGraph // restricted by regions in the list if given, // by systems in the list if given, // or to within a given area if placeradius is given - void write_subgraphs_tmg(std::vector &graph_vector, std::string path, size_t graphnum, unsigned int threadnum, WaypointQuadtree *qt) + void write_subgraphs_tmg( std::vector &graph_vector, std::string path, size_t graphnum, + unsigned int threadnum, WaypointQuadtree *qt, ElapsedTime *et) { unsigned int cv_count, tv_count; std::ofstream simplefile((path+graph_vector[graphnum].filename()).data()); std::ofstream collapfile((path+graph_vector[graphnum+1].filename()).data()); @@ -508,6 +509,10 @@ class HighwayGraph { t->traveler_num[threadnum] = travnum; travnum++; } + #ifdef threading_enabled + if (graph_vector[graphnum].cat != graph_vector[graphnum-1].cat) + std::cout << '\n' + et->et() + "Writing " + graph_vector[graphnum].category() + " graphs.\n"; + #endif std::cout << graph_vector[graphnum].tag() << '(' << mv.size() << ',' << mse.size() << ") " << '(' << cv_count << ',' << mce.size() << ") " diff --git a/siteupdate/cplusplus/functions/graph_generation.cpp b/siteupdate/cplusplus/functions/graph_generation.cpp index 9798c4cd..825728f7 100644 --- a/siteupdate/cplusplus/functions/graph_generation.cpp +++ b/siteupdate/cplusplus/functions/graph_generation.cpp @@ -53,336 +53,27 @@ else { list *regions; graph_types.push_back({"master", "All Travel Mapping Data", "These graphs contain all routes currently plotted in the Travel Mapping project."}); size_t graphnum = 3; - - - // graphs restricted by place/area - from areagraphs.csv file - cout << '\n' << et.et() << "Creating area data graphs." << endl; - filename = args.highwaydatapath+"/graphs/areagraphs.csv"; - file.open(filename.data()); - getline(file, line); // ignore header line - list area_list; - while (getline(file, line)) - { vector fields; - char *cline = new char[line.size()+1]; - strcpy(cline, line.data()); - for (char *token = strtok(cline, ";"); token; token = strtok(0, ";")) fields.push_back(token); - if (fields.size() != 5) - { cout << "Could not parse areagraphs.csv line: " << cline << endl; - delete[] cline; - continue; - } - area_list.emplace_back(fields[0], fields[1], fields[2], fields[3], fields[4]); - delete[] cline; - } - file.close(); - - // add entries to graph_vector - for (PlaceRadius &a : area_list) - { graph_vector.emplace_back(a.base + to_string(a.r) + "-area", a.place + " (" + to_string(a.r) + " mi radius)", - 's', 'a', (list*)0, (list*)0, &a); - graph_vector.emplace_back(a.base + to_string(a.r) + "-area", a.place + " (" + to_string(a.r) + " mi radius)", - 'c', 'a', (list*)0, (list*)0, &a); - graph_vector.emplace_back(a.base + to_string(a.r) + "-area", a.place + " (" + to_string(a.r) + " mi radius)", - 't', 'a', (list*)0, (list*)0, &a); - } - // write new graph_vector entries to disk #ifdef threading_enabled - // set up for threaded subgraph generation - for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &all_waypoints); - for (unsigned int t = 0; t < args.numthreads; t++) - thr[t]->join(); - for (unsigned int t = 0; t < args.numthreads; t++) - delete thr[t]; - #else - while (graphnum < graph_vector.size()) - { graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &all_waypoints); - graphnum += 3; - } + cout << et.et() << "Setting up subgraphs." << flush; #endif - graph_types.push_back({"area", "Routes Within a Given Radius of a Place", - "These graphs contain all routes currently plotted within the given distance radius of the given place."}); - cout << '!' << endl; - area_list.clear(); - - - // Graphs restricted by region - cout << et.et() << "Creating regional data graphs." << endl; - - // We will create graph data and a graph file for each region that includes - // any active or preview systems - - // add entries to graph_vector - for (Region ®ion : all_regions) - { if (region.active_preview_mileage == 0) continue; - regions = new list(1, ®ion); - // deleted on termination of program - graph_vector.emplace_back(region.code + "-region", region.name + " (" + region.type + ")", - 's', 'r', regions, (list*)0, (PlaceRadius*)0); - graph_vector.emplace_back(region.code + "-region", region.name + " (" + region.type + ")", - 'c', 'r', (list*)0, (list*)0, (PlaceRadius*)0); - graph_vector.emplace_back(region.code + "-region", region.name + " (" + region.type + ")", - 't', 'r', (list*)0, (list*)0, (PlaceRadius*)0); - } - // write new graph_vector entries to disk + #include "subgraphs/continent.cpp" + #include "subgraphs/multisystem.cpp" + #include "subgraphs/system.cpp" + #include "subgraphs/country.cpp" + #include "subgraphs/multiregion.cpp" + #include "subgraphs/area.cpp" + #include "subgraphs/region.cpp" #ifdef threading_enabled - // set up for threaded subgraph generation - for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &all_waypoints); - for (unsigned int t = 0; t < args.numthreads; t++) - thr[t]->join(); - for (unsigned int t = 0; t < args.numthreads; t++) - delete thr[t]; - #else - while (graphnum < graph_vector.size()) - { graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &all_waypoints); - graphnum += 3; - } - #endif - graph_types.push_back({"region", "Routes Within a Single Region", "These graphs contain all routes currently plotted within the given region."}); - cout << "!" << endl; - - - // Graphs restricted by system - from systemgraphs.csv file - cout << et.et() << "Creating system data graphs." << endl; - - // We will create graph data and a graph file for only a few interesting - // systems, as many are not useful on their own - HighwaySystem *h; - filename = args.highwaydatapath+"/graphs/systemgraphs.csv"; - file.open(filename.data()); - getline(file, line); // ignore header line - - // add entries to graph_vector - while (getline(file, line)) - { h = 0; - for (HighwaySystem *hs : highway_systems) - if (hs->systemname == line) - { h = hs; - break; - } - if (h) - { systems = new list(1, h); - // deleted on termination of program - graph_vector.emplace_back(h->systemname + "-system", h->systemname + " (" + h->fullname + ")", - 's', 's', (list*)0, systems, (PlaceRadius*)0); - graph_vector.emplace_back(h->systemname + "-system", h->systemname + " (" + h->fullname + ")", - 'c', 's', (list*)0, (list*)0, (PlaceRadius*)0); - graph_vector.emplace_back(h->systemname + "-system", h->systemname + " (" + h->fullname + ")", - 't', 's', (list*)0, (list*)0, (PlaceRadius*)0); - } - } - file.close(); // write new graph_vector entries to disk - #ifdef threading_enabled // set up for threaded subgraph generation for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &all_waypoints); + thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &all_waypoints, &et); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); for (unsigned int t = 0; t < args.numthreads; t++) delete thr[t]; - #else - while (graphnum < graph_vector.size()) - { graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &all_waypoints); - graphnum += 3; - } - #endif - if (h) graph_types.push_back({"system", "Routes Within a Single Highway System", - "These graphs contain the routes within a single highway system and are not restricted by region."}); - cout << "!" << endl; - - - // Some additional interesting graphs, the "multisystem" graphs - cout << et.et() << "Creating multisystem graphs." << endl; - - filename = args.highwaydatapath+"/graphs/multisystem.csv"; - file.open(filename.data()); - getline(file, line); // ignore header line - - // add entries to graph_vector - while (getline(file, line)) - { vector fields; - char *cline = new char[line.size()+1]; - strcpy(cline, line.data()); - for (char *token = strtok(cline, ";"); token; token = strtok(0, ";")) fields.push_back(token); - if (fields.size() != 3) - { cout << "Could not parse multisystem.csv line: " << line << endl; - delete[] cline; - continue; - } - systems = new list; - // deleted on termination of program - for(char* s = strtok(fields[2], ","); s; s = strtok(0, ",")) - for (HighwaySystem *h : highway_systems) - if (s == h->systemname) - { systems->push_back(h); - break; - } - graph_vector.emplace_back(fields[1], fields[0], - 's', 'S', (list*)0, systems, (PlaceRadius*)0); - graph_vector.emplace_back(fields[1], fields[0], - 'c', 'S', (list*)0, (list*)0, (PlaceRadius*)0); - graph_vector.emplace_back(fields[1], fields[0], - 't', 'S', (list*)0, (list*)0, (PlaceRadius*)0); - delete[] cline; - } - file.close(); - // write new graph_vector entries to disk - #ifdef threading_enabled - // set up for threaded subgraph generation - for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &all_waypoints); - for (unsigned int t = 0; t < args.numthreads; t++) - thr[t]->join(); - for (unsigned int t = 0; t < args.numthreads; t++) - delete thr[t]; - #else - while (graphnum < graph_vector.size()) - { graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &all_waypoints); - graphnum += 3; - } - #endif - graph_types.push_back({"multisystem", "Routes Within Multiple Highway Systems", "These graphs contain the routes within a set of highway systems."}); - cout << "!" << endl; - - - // Some additional interesting graphs, the "multiregion" graphs - cout << et.et() << "Creating multiregion graphs." << endl; - - filename = args.highwaydatapath+"/graphs/multiregion.csv"; - file.open(filename.data()); - getline(file, line); // ignore header line - - // add entries to graph_vector - while (getline(file, line)) - { vector fields; - char *cline = new char[line.size()+1]; - strcpy(cline, line.data()); - for (char *token = strtok(cline, ";"); token; token = strtok(0, ";")) fields.push_back(token); - if (fields.size() != 3) - { cout << "Could not parse multiregion.csv line: " << line << endl; - delete[] cline; - continue; - } - regions = new list; - // deleted on termination of program - for(char* rg = strtok(fields[2], ","); rg; rg = strtok(0, ",")) - for (Region &r : all_regions) - if (rg == r.code) - { regions->push_back(&r); - break; - } - graph_vector.emplace_back(fields[1], fields[0], - 's', 'R', regions, (list*)0, (PlaceRadius*)0); - graph_vector.emplace_back(fields[1], fields[0], - 'c', 'R', (list*)0, (list*)0, (PlaceRadius*)0); - graph_vector.emplace_back(fields[1], fields[0], - 't', 'R', (list*)0, (list*)0, (PlaceRadius*)0); - delete[] cline; - } - file.close(); - // write new graph_vector entries to disk - #ifdef threading_enabled - // set up for threaded subgraph generation - for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &all_waypoints); - for (unsigned int t = 0; t < args.numthreads; t++) - thr[t]->join(); - for (unsigned int t = 0; t < args.numthreads; t++) - delete thr[t]; - #else - while (graphnum < graph_vector.size()) - { graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &all_waypoints); - graphnum += 3; - } - #endif - graph_types.push_back({"multiregion", "Routes Within Multiple Regions", "These graphs contain the routes within a set of regions."}); - cout << "!" << endl; - - - // country graphs - we find countries that have regions - // that have routes with active or preview mileage - cout << et.et() << "Creating country graphs." << endl; - - // add entries to graph_vector - for (pair &c : countries) - { regions = new list; - // deleted on termination of program - for (Region &r : all_regions) - // does it match this country and have routes? - if (&c == r.country && r.active_preview_mileage) - regions->push_back(&r); - // does it have at least two? if none, no data, - // if 1 we already generated a graph for that one region - if (regions->size() < 2) delete regions; - else { graph_vector.emplace_back(c.first + "-country", c.second + " All Routes in Country", - 's', 'c', regions, (list*)0, (PlaceRadius*)0); - graph_vector.emplace_back(c.first + "-country", c.second + " All Routes in Country", - 'c', 'c', (list*)0, (list*)0, (PlaceRadius*)0); - graph_vector.emplace_back(c.first + "-country", c.second + " All Routes in Country", - 't', 't', (list*)0, (list*)0, (PlaceRadius*)0); - } - } - // write new graph_vector entries to disk - #ifdef threading_enabled - // set up for threaded subgraph generation - for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &all_waypoints); - for (unsigned int t = 0; t < args.numthreads; t++) - thr[t]->join(); - for (unsigned int t = 0; t < args.numthreads; t++) - delete thr[t]; - #else - while (graphnum < graph_vector.size()) - { graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &all_waypoints); - graphnum += 3; - } - #endif - graph_types.push_back({"country", "Routes Within a Single Multi-Region Country", - "These graphs contain the routes within a single country that is composed of multiple regions that contain plotted routes. " + - string("Countries consisting of a single region are represented by their regional graph.")}); - cout << "!" << endl; - - - // continent graphs -- any continent with data will be created - cout << et.et() << "Creating continent graphs." << endl; - - // add entries to graph_vector - for (pair &c : continents) - { regions = new list; - // deleted on termination of program - for (Region &r : all_regions) - // does it match this continent and have routes? - if (&c == r.continent && r.active_preview_mileage) - regions->push_back(&r); - // generate for any continent with at least 1 region with mileage - if (regions->size() < 1) delete regions; - else { graph_vector.emplace_back(c.first + "-continent", c.second + " All Routes on Continent", - 's', 'C', regions, (list*)0, (PlaceRadius*)0); - graph_vector.emplace_back(c.first + "-continent", c.second + " All Routes on Continent", - 'c', 'C', (list*)0, (list*)0, (PlaceRadius*)0); - graph_vector.emplace_back(c.first + "-continent", c.second + " All Routes on Continent", - 't', 'C', (list*)0, (list*)0, (PlaceRadius*)0); - } - } - // write new graph_vector entries to disk - #ifdef threading_enabled - // set up for threaded subgraph generation - for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &all_waypoints); - for (unsigned int t = 0; t < args.numthreads; t++) - thr[t]->join(); - for (unsigned int t = 0; t < args.numthreads; t++) - delete thr[t]; - #else - while (graphnum < graph_vector.size()) - { graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &all_waypoints); - graphnum += 3; - } + cout << '!' << endl; #endif - graph_types.push_back({"continent", "Routes Within a Continent", "These graphs contain the routes on a continent."}); - cout << "!" << endl; } graph_data.clear(); diff --git a/siteupdate/cplusplus/functions/subgraphs/area.cpp b/siteupdate/cplusplus/functions/subgraphs/area.cpp new file mode 100644 index 00000000..f89c9612 --- /dev/null +++ b/siteupdate/cplusplus/functions/subgraphs/area.cpp @@ -0,0 +1,42 @@ +// graphs restricted by place/area - from areagraphs.csv file +#ifndef threading_enabled +cout << et.et() << "Creating area data graphs." << endl; +#endif +filename = args.highwaydatapath+"/graphs/areagraphs.csv"; +file.open(filename.data()); +getline(file, line); // ignore header line +list area_list; +while (getline(file, line)) +{ vector fields; + char *cline = new char[line.size()+1]; + strcpy(cline, line.data()); + for (char *token = strtok(cline, ";"); token; token = strtok(0, ";")) fields.push_back(token); + if (fields.size() != 5) + { cout << "Could not parse areagraphs.csv line: " << cline << endl; + delete[] cline; + continue; + } + area_list.emplace_back(fields[0], fields[1], fields[2], fields[3], fields[4]); + delete[] cline; +} +file.close(); + +// add entries to graph_vector +for (PlaceRadius &a : area_list) +{ graph_vector.emplace_back(a.base + to_string(a.r) + "-area", a.place + " (" + to_string(a.r) + " mi radius)", + 's', 'a', (list*)0, (list*)0, &a); + graph_vector.emplace_back(a.base + to_string(a.r) + "-area", a.place + " (" + to_string(a.r) + " mi radius)", + 'c', 'a', (list*)0, (list*)0, &a); + graph_vector.emplace_back(a.base + to_string(a.r) + "-area", a.place + " (" + to_string(a.r) + " mi radius)", + 't', 'a', (list*)0, (list*)0, &a); +} +#ifndef threading_enabled +// write new graph_vector entries to disk +while (graphnum < graph_vector.size()) +{ graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &all_waypoints, &et); + graphnum += 3; +} +cout << '!' << endl; +#endif +graph_types.push_back({"area", "Routes Within a Given Radius of a Place", + "These graphs contain all routes currently plotted within the given distance radius of the given place."}); diff --git a/siteupdate/cplusplus/functions/subgraphs/continent.cpp b/siteupdate/cplusplus/functions/subgraphs/continent.cpp new file mode 100644 index 00000000..00d9a732 --- /dev/null +++ b/siteupdate/cplusplus/functions/subgraphs/continent.cpp @@ -0,0 +1,31 @@ +// continent graphs -- any continent with data will be created +#ifndef threading_enabled +cout << et.et() << "Creating continent graphs." << endl; +#endif +// add entries to graph_vector +for (pair &c : continents) +{ regions = new list; + // deleted on termination of program + for (Region &r : all_regions) + // does it match this continent and have routes? + if (&c == r.continent && r.active_preview_mileage) + regions->push_back(&r); + // generate for any continent with at least 1 region with mileage + if (regions->size() < 1) delete regions; + else { graph_vector.emplace_back(c.first + "-continent", c.second + " All Routes on Continent", + 's', 'C', regions, (list*)0, (PlaceRadius*)0); + graph_vector.emplace_back(c.first + "-continent", c.second + " All Routes on Continent", + 'c', 'C', (list*)0, (list*)0, (PlaceRadius*)0); + graph_vector.emplace_back(c.first + "-continent", c.second + " All Routes on Continent", + 't', 'C', (list*)0, (list*)0, (PlaceRadius*)0); + } +} +#ifndef threading_enabled +// write new graph_vector entries to disk +while (graphnum < graph_vector.size()) +{ graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &all_waypoints, &et); + graphnum += 3; +} +cout << "!" << endl; +#endif +graph_types.push_back({"continent", "Routes Within a Continent", "These graphs contain the routes on a continent."}); diff --git a/siteupdate/cplusplus/functions/subgraphs/country.cpp b/siteupdate/cplusplus/functions/subgraphs/country.cpp new file mode 100644 index 00000000..4ef8ec30 --- /dev/null +++ b/siteupdate/cplusplus/functions/subgraphs/country.cpp @@ -0,0 +1,35 @@ +// country graphs - we find countries that have regions +// that have routes with active or preview mileage +#ifndef threading_enabled +cout << et.et() << "Creating country graphs." << endl; +#endif +// add entries to graph_vector +for (pair &c : countries) +{ regions = new list; + // deleted on termination of program + for (Region &r : all_regions) + // does it match this country and have routes? + if (&c == r.country && r.active_preview_mileage) + regions->push_back(&r); + // does it have at least two? if none, no data, + // if 1 we already generated a graph for that one region + if (regions->size() < 2) delete regions; + else { graph_vector.emplace_back(c.first + "-country", c.second + " All Routes in Country", + 's', 'c', regions, (list*)0, (PlaceRadius*)0); + graph_vector.emplace_back(c.first + "-country", c.second + " All Routes in Country", + 'c', 'c', (list*)0, (list*)0, (PlaceRadius*)0); + graph_vector.emplace_back(c.first + "-country", c.second + " All Routes in Country", + 't', 'c', (list*)0, (list*)0, (PlaceRadius*)0); + } +} +#ifndef threading_enabled +// write new graph_vector entries to disk +while (graphnum < graph_vector.size()) +{ graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &all_waypoints, &et); + graphnum += 3; +} +cout << "!" << endl; +#endif +graph_types.push_back({"country", "Routes Within a Single Multi-Region Country", + "These graphs contain the routes within a single country that is composed of multiple regions that contain plotted routes. " + + string("Countries consisting of a single region are represented by their regional graph.")}); diff --git a/siteupdate/cplusplus/functions/subgraphs/multiregion.cpp b/siteupdate/cplusplus/functions/subgraphs/multiregion.cpp new file mode 100644 index 00000000..a6e0bc44 --- /dev/null +++ b/siteupdate/cplusplus/functions/subgraphs/multiregion.cpp @@ -0,0 +1,45 @@ +// Some additional interesting graphs, the "multiregion" graphs +#ifndef threading_enabled +cout << et.et() << "Creating multiregion graphs." << endl; +#endif +filename = args.highwaydatapath+"/graphs/multiregion.csv"; +file.open(filename.data()); +getline(file, line); // ignore header line + +// add entries to graph_vector +while (getline(file, line)) +{ vector fields; + char *cline = new char[line.size()+1]; + strcpy(cline, line.data()); + for (char *token = strtok(cline, ";"); token; token = strtok(0, ";")) fields.push_back(token); + if (fields.size() != 3) + { cout << "Could not parse multiregion.csv line: " << line << endl; + delete[] cline; + continue; + } + regions = new list; + // deleted on termination of program + for(char* rg = strtok(fields[2], ","); rg; rg = strtok(0, ",")) + for (Region &r : all_regions) + if (rg == r.code) + { regions->push_back(&r); + break; + } + graph_vector.emplace_back(fields[1], fields[0], + 's', 'R', regions, (list*)0, (PlaceRadius*)0); + graph_vector.emplace_back(fields[1], fields[0], + 'c', 'R', (list*)0, (list*)0, (PlaceRadius*)0); + graph_vector.emplace_back(fields[1], fields[0], + 't', 'R', (list*)0, (list*)0, (PlaceRadius*)0); + delete[] cline; +} +file.close(); +#ifndef threading_enabled +// write new graph_vector entries to disk +while (graphnum < graph_vector.size()) +{ graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &all_waypoints, &et); + graphnum += 3; +} +cout << "!" << endl; +#endif +graph_types.push_back({"multiregion", "Routes Within Multiple Regions", "These graphs contain the routes within a set of regions."}); diff --git a/siteupdate/cplusplus/functions/subgraphs/multisystem.cpp b/siteupdate/cplusplus/functions/subgraphs/multisystem.cpp new file mode 100644 index 00000000..3a053cd7 --- /dev/null +++ b/siteupdate/cplusplus/functions/subgraphs/multisystem.cpp @@ -0,0 +1,45 @@ +// Some additional interesting graphs, the "multisystem" graphs +#ifndef threading_enabled +cout << et.et() << "Creating multisystem graphs." << endl; +#endif +filename = args.highwaydatapath+"/graphs/multisystem.csv"; +file.open(filename.data()); +getline(file, line); // ignore header line + +// add entries to graph_vector +while (getline(file, line)) +{ vector fields; + char *cline = new char[line.size()+1]; + strcpy(cline, line.data()); + for (char *token = strtok(cline, ";"); token; token = strtok(0, ";")) fields.push_back(token); + if (fields.size() != 3) + { cout << "Could not parse multisystem.csv line: " << line << endl; + delete[] cline; + continue; + } + systems = new list; + // deleted on termination of program + for(char* s = strtok(fields[2], ","); s; s = strtok(0, ",")) + for (HighwaySystem *h : highway_systems) + if (s == h->systemname) + { systems->push_back(h); + break; + } + graph_vector.emplace_back(fields[1], fields[0], + 's', 'S', (list*)0, systems, (PlaceRadius*)0); + graph_vector.emplace_back(fields[1], fields[0], + 'c', 'S', (list*)0, (list*)0, (PlaceRadius*)0); + graph_vector.emplace_back(fields[1], fields[0], + 't', 'S', (list*)0, (list*)0, (PlaceRadius*)0); + delete[] cline; +} +file.close(); +#ifndef threading_enabled +// write new graph_vector entries to disk +while (graphnum < graph_vector.size()) +{ graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &all_waypoints, &et); + graphnum += 3; +} +cout << "!" << endl; +#endif +graph_types.push_back({"multisystem", "Routes Within Multiple Highway Systems", "These graphs contain the routes within a set of highway systems."}); diff --git a/siteupdate/cplusplus/functions/subgraphs/region.cpp b/siteupdate/cplusplus/functions/subgraphs/region.cpp new file mode 100644 index 00000000..9d986559 --- /dev/null +++ b/siteupdate/cplusplus/functions/subgraphs/region.cpp @@ -0,0 +1,28 @@ +// Graphs restricted by region +#ifndef threading_enabled +cout << et.et() << "Creating regional data graphs." << endl; +#endif +// We will create graph data and a graph file for each region that includes +// any active or preview systems + +// add entries to graph_vector +for (Region ®ion : all_regions) +{ if (region.active_preview_mileage == 0) continue; + regions = new list(1, ®ion); + // deleted on termination of program + graph_vector.emplace_back(region.code + "-region", region.name + " (" + region.type + ")", + 's', 'r', regions, (list*)0, (PlaceRadius*)0); + graph_vector.emplace_back(region.code + "-region", region.name + " (" + region.type + ")", + 'c', 'r', (list*)0, (list*)0, (PlaceRadius*)0); + graph_vector.emplace_back(region.code + "-region", region.name + " (" + region.type + ")", + 't', 'r', (list*)0, (list*)0, (PlaceRadius*)0); +} +#ifndef threading_enabled +// write new graph_vector entries to disk +while (graphnum < graph_vector.size()) +{ graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &all_waypoints, &et); + graphnum += 3; +} +cout << "!" << endl; +#endif +graph_types.push_back({"region", "Routes Within a Single Region", "These graphs contain all routes currently plotted within the given region."}); diff --git a/siteupdate/cplusplus/functions/subgraphs/system.cpp b/siteupdate/cplusplus/functions/subgraphs/system.cpp new file mode 100644 index 00000000..58445997 --- /dev/null +++ b/siteupdate/cplusplus/functions/subgraphs/system.cpp @@ -0,0 +1,41 @@ +// Graphs restricted by system - from systemgraphs.csv file +#ifndef threading_enabled +cout << et.et() << "Creating system data graphs." << endl; +#endif +// We will create graph data and a graph file for only a few interesting +// systems, as many are not useful on their own +HighwaySystem *h; +filename = args.highwaydatapath+"/graphs/systemgraphs.csv"; +file.open(filename.data()); +getline(file, line); // ignore header line + +// add entries to graph_vector +while (getline(file, line)) +{ h = 0; + for (HighwaySystem *hs : highway_systems) + if (hs->systemname == line) + { h = hs; + break; + } + if (h) + { systems = new list(1, h); + // deleted on termination of program + graph_vector.emplace_back(h->systemname + "-system", h->systemname + " (" + h->fullname + ")", + 's', 's', (list*)0, systems, (PlaceRadius*)0); + graph_vector.emplace_back(h->systemname + "-system", h->systemname + " (" + h->fullname + ")", + 'c', 's', (list*)0, (list*)0, (PlaceRadius*)0); + graph_vector.emplace_back(h->systemname + "-system", h->systemname + " (" + h->fullname + ")", + 't', 's', (list*)0, (list*)0, (PlaceRadius*)0); + } +} +file.close(); +#ifndef threading_enabled +// write new graph_vector entries to disk +while (graphnum < graph_vector.size()) +{ graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &all_waypoints, &et); + graphnum += 3; +} +cout << "!" << endl; +#endif +if (h) graph_types.push_back({"system", "Routes Within a Single Highway System", + "These graphs contain the routes within a single highway system and are not restricted by region."}); diff --git a/siteupdate/cplusplus/threads/SubgraphThread.cpp b/siteupdate/cplusplus/threads/SubgraphThread.cpp index 7f256d86..665b50e7 100644 --- a/siteupdate/cplusplus/threads/SubgraphThread.cpp +++ b/siteupdate/cplusplus/threads/SubgraphThread.cpp @@ -1,6 +1,6 @@ void SubgraphThread ( unsigned int id, HighwayGraph *graph_data, std::vector *graph_vector, - size_t *index, std::mutex *mtx, std::string path, WaypointQuadtree *qt + size_t *index, std::mutex *mtx, std::string path, WaypointQuadtree *qt, ElapsedTime *et ) { //std::cout << "Starting SubgraphThread " << id << std::endl; while (*index < graph_vector->size()) @@ -14,6 +14,6 @@ void SubgraphThread size_t i = *index; *index += 3; mtx->unlock(); - graph_data->write_subgraphs_tmg(*graph_vector, path, i, id, qt); + graph_data->write_subgraphs_tmg(*graph_vector, path, i, id, qt, et); } } From 835bd728c0a8acd6dbd3c31a9d4909ccc3e285dd Mon Sep 17 00:00:00 2001 From: eric bryant Date: Sun, 23 Jun 2019 19:03:27 -0400 Subject: [PATCH 2/2] threads: "modify lists" -> "modify iterators" --- siteupdate/cplusplus/siteupdate.cpp | 27 ++++++++++--------- .../cplusplus/threads/ComputeStatsThread.cpp | 17 ++++++------ .../cplusplus/threads/ConcAugThread.cpp | 18 ++++++------- .../cplusplus/threads/NmpMergedThread.cpp | 17 ++++++------ .../cplusplus/threads/ReadListThread.cpp | 21 ++++++++------- .../cplusplus/threads/ReadWptThread.cpp | 19 +++++++------ .../cplusplus/threads/UserLogThread.cpp | 21 ++++++++------- 7 files changed, 73 insertions(+), 67 deletions(-) diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index 59c3c0f4..969ca834 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -292,11 +292,11 @@ int main(int argc, char *argv[]) cout << et.et() << "Reading waypoints for all routes." << endl; #ifdef threading_enabled // set up for threaded processing of highway systems - list hs_list = highway_systems; + list::iterator hs_it = highway_systems.begin(); thread **thr = new thread*[args.numthreads]; for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(ReadWptThread, t, &hs_list, &list_mtx, args.highwaydatapath+"/hwy_data", + thr[t] = new thread(ReadWptThread, t, &highway_systems, &hs_it, &list_mtx, args.highwaydatapath+"/hwy_data", &el, &all_wpt_files, &all_waypoints, &strtok_mtx, datacheckerrors); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); @@ -373,9 +373,9 @@ int main(int argc, char *argv[]) { cout << et.et() << "Writing near-miss point merged wpt files." << endl; //FIXME output dots to indicate progress #ifdef threading_enabled // set up for threaded nmp_merged file writes - hs_list = highway_systems; + hs_it = highway_systems.begin(); for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(NmpMergedThread, &hs_list, &list_mtx, &args.nmpmergepath); + thr[t] = new thread(NmpMergedThread, t, &highway_systems, &hs_it, &list_mtx, &args.nmpmergepath); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); for (unsigned int t = 0; t < args.numthreads; t++) @@ -400,12 +400,13 @@ int main(int argc, char *argv[]) // Create a list of TravelerList objects, one per person list traveler_lists; + list::iterator id_it = traveler_ids.begin(); cout << et.et() << "Processing traveler list files:"; #ifdef threading_enabled // set up for threaded .list file processing for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(ReadListThread, &traveler_ids, &traveler_lists, &list_mtx, &strtok_mtx, &args, &route_hash); + thr[t] = new thread(ReadListThread, t, &traveler_ids, &id_it, &traveler_lists, &list_mtx, &strtok_mtx, &args, &route_hash); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); for (unsigned int t = 0; t < args.numthreads; t++) @@ -592,10 +593,10 @@ int main(int argc, char *argv[]) //#include "debug/concurrency_augments.cpp" #ifdef threading_enabled // set up for threaded concurrency augments - list travlists_copy = traveler_lists; + list::iterator tl_it = traveler_lists.begin(); for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(ConcAugThread, t, &travlists_copy, &list_mtx, &log_mtx, &concurrencyfile); + thr[t] = new thread(ConcAugThread, t, &traveler_lists, &tl_it, &list_mtx, &log_mtx, &concurrencyfile); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); for (unsigned int t = 0; t < args.numthreads; t++) @@ -631,9 +632,9 @@ int main(int argc, char *argv[]) #ifdef threading_enabled // set up for threaded stats computation - hs_list = highway_systems; + hs_it = highway_systems.begin(); for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(ComputeStatsThread, t, &hs_list, &list_mtx); + thr[t] = new thread(ComputeStatsThread, t, &highway_systems, &hs_it, &list_mtx); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); for (unsigned int t = 0; t < args.numthreads; t++) @@ -723,10 +724,12 @@ int main(int argc, char *argv[]) cout << et.et() << "Creating per-traveler stats logs and augmenting data structure." << flush; #ifdef threading_enabled // set up for threaded user logs - list tl_list = traveler_lists; + tl_it = traveler_lists.begin(); for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread \ - (UserLogThread, t, &tl_list, &list_mtx, &clin_db_val, active_only_miles, active_preview_miles, &highway_systems, args.logfilepath+"/users/"); + thr[t] = new thread + ( UserLogThread, t, &traveler_lists, &tl_it, &list_mtx, &clin_db_val, active_only_miles, + active_preview_miles, &highway_systems, args.logfilepath+"/users/" + ); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); for (unsigned int t = 0; t < args.numthreads; t++) diff --git a/siteupdate/cplusplus/threads/ComputeStatsThread.cpp b/siteupdate/cplusplus/threads/ComputeStatsThread.cpp index 92494a8f..78d6dd59 100644 --- a/siteupdate/cplusplus/threads/ComputeStatsThread.cpp +++ b/siteupdate/cplusplus/threads/ComputeStatsThread.cpp @@ -1,16 +1,15 @@ -void ComputeStatsThread(unsigned int id, std::list *hs_list, std::mutex *mtx) -{ //std::cout << "Starting ComputeStatsThread " << id << std::endl; - while (hs_list->size()) +void ComputeStatsThread(unsigned int id, std::list *hs_list, std::list::iterator *it, std::mutex *mtx) +{ //printf("Starting ComputeStatsThread %02i\n", id); fflush(stdout); + while (*it != hs_list->end()) { mtx->lock(); - if (!hs_list->size()) + if (*it == hs_list->end()) { mtx->unlock(); return; } - //std::cout << "Thread " << id << " with hs_list->size()=" << hs_list->size() << std::endl; - HighwaySystem *h(hs_list->front()); - //std::cout << "Thread " << id << " assigned " << h->systemname << std::endl; - hs_list->pop_front(); - //std::cout << "Thread " << id << " hs_list->pop_front() successful." << std::endl; + HighwaySystem *h(**it); + //printf("ComputeStatsThread %02i assigned %s\n", id, h->systemname.data()); fflush(stdout); + (*it)++; + //printf("ComputeStatsThread %02i (*it)++ OK. Releasing lock.\n", id); fflush(stdout); mtx->unlock(); std::cout << '.' << std::flush; for (Route &r : h->route_list) diff --git a/siteupdate/cplusplus/threads/ConcAugThread.cpp b/siteupdate/cplusplus/threads/ConcAugThread.cpp index d43bba09..c7f3cdb9 100644 --- a/siteupdate/cplusplus/threads/ConcAugThread.cpp +++ b/siteupdate/cplusplus/threads/ConcAugThread.cpp @@ -1,16 +1,16 @@ -void ConcAugThread(unsigned int id, std::list *travlists, std::mutex *tl_mtx, std::mutex *log_mtx, std::ofstream *concurrencyfile) -{ //std::cout << "Starting NMPMergedThread " << id << std::endl; - while (travlists->size()) +void ConcAugThread(unsigned int id, std::list *travlists, std::list::iterator *it, + std::mutex *tl_mtx, std::mutex *log_mtx, std::ofstream *concurrencyfile) +{ //printf("Starting ConcAugThread %02i\n", id); fflush(stdout); + while (*it != travlists->end()) { tl_mtx->lock(); - if (!travlists->size()) + if (*it == travlists->end()) { tl_mtx->unlock(); return; } - //std::cout << "Thread " << id << " with travlists.size()=" << travlists->size() << std::endl; - TravelerList *t(travlists->front()); - //std::cout << "Thread " << id << " assigned " << t->traveler_name << std::endl; - travlists->pop_front(); - //std::cout << "Thread " << id << " travlists->pop_front() successful." << std::endl; + TravelerList *t(**it); + //printf("ConcAugThread %02i assigned %s\n", id, t->traveler_name.data()); fflush(stdout); + (*it)++; + //printf("ConcAugThread %02i (*it)++\n", id); fflush(stdout); tl_mtx->unlock(); std::cout << '.' << std::flush; for (HighwaySegment *s : t->clinched_segments) diff --git a/siteupdate/cplusplus/threads/NmpMergedThread.cpp b/siteupdate/cplusplus/threads/NmpMergedThread.cpp index 1dad34e3..909c958d 100644 --- a/siteupdate/cplusplus/threads/NmpMergedThread.cpp +++ b/siteupdate/cplusplus/threads/NmpMergedThread.cpp @@ -1,16 +1,15 @@ -void NmpMergedThread(std::list *hs_list, std::mutex *mtx, std::string *nmpmergepath) -{ //std::cout << "Starting NMPMergedThread " << id << std::endl; - while (hs_list->size()) +void NmpMergedThread(unsigned int id, std::list *hs_list, std::list::iterator *it, std::mutex *mtx, std::string *nmpmergepath) +{ //printf("Starting NMPMergedThread %02i\n", id); fflush(stdout); + while (*it != hs_list->end()) { mtx->lock(); - if (!hs_list->size()) + if (*it == hs_list->end()) { mtx->unlock(); return; } - //std::cout << "Thread " << id << " with hs_list.size()=" << hs_list.size() << std::endl; - HighwaySystem *h(hs_list->front()); - //std::cout << "Thread " << id << " assigned " << h->systemname << std::endl; - hs_list->pop_front(); - //std::cout << "Thread " << id << " hs_list->pop_front() successful." << std::endl; + HighwaySystem *h(**it); + //printf("NmpMergedThread %02i assigned %s\n", id, h->systemname.data()); fflush(stdout); + (*it)++; + //printf("NmpMergedThread %02i (*it)++\n", id); fflush(stdout); mtx->unlock(); for (Route &r : h->route_list) r.write_nmp_merged(*nmpmergepath + "/" + r.region->code); diff --git a/siteupdate/cplusplus/threads/ReadListThread.cpp b/siteupdate/cplusplus/threads/ReadListThread.cpp index f990922e..748bc590 100644 --- a/siteupdate/cplusplus/threads/ReadListThread.cpp +++ b/siteupdate/cplusplus/threads/ReadListThread.cpp @@ -1,16 +1,19 @@ -void ReadListThread(std::list *traveler_ids, std::list *traveler_lists, std::mutex *tl_mtx, std::mutex *strtok_mtx, Arguments *args, std::unordered_map *route_hash) -{ //std::cout << "Starting ReadListThread " << id << std::endl; - while (traveler_ids->size()) +void ReadListThread +( unsigned int id, std::list *traveler_ids, std::list::iterator *it, + std::list *traveler_lists, std::mutex *tl_mtx, std::mutex *strtok_mtx, + Arguments *args, std::unordered_map *route_hash +) +{ //printf("Starting ReadListThread %02i\n", id); fflush(stdout); + while (*it != traveler_ids->end()) { tl_mtx->lock(); - if (!traveler_ids->size()) + if (*it == traveler_ids->end()) { tl_mtx->unlock(); return; } - //std::cout << "Thread " << id << " with traveler_ids.size()=" << traveler_ids.size() << std::endl; - std::string tl(traveler_ids->front()); - //std::cout << "Thread " << id << " assigned " << tl << std::endl; - traveler_ids->pop_front(); - //std::cout << "Thread " << id << " traveler_ids->pop_front() successful." << std::endl; + std::string tl(**it); + //printf("ReadListThread %02i assigned %s\n", id, tl.data()); fflush(stdout); + (*it)++; + //printf("ReadListThread %02i (*it)++\n", id); fflush(stdout); std::cout << ' ' << tl << std::flush; tl_mtx->unlock(); TravelerList *t = new TravelerList(tl, route_hash, args, strtok_mtx); diff --git a/siteupdate/cplusplus/threads/ReadWptThread.cpp b/siteupdate/cplusplus/threads/ReadWptThread.cpp index 8a1143c5..e4c00de8 100644 --- a/siteupdate/cplusplus/threads/ReadWptThread.cpp +++ b/siteupdate/cplusplus/threads/ReadWptThread.cpp @@ -1,20 +1,19 @@ void ReadWptThread -( unsigned int id, std::list *hs_list, std::mutex *hs_mtx, - std::string path, ErrorList *el, std::unordered_set *all_wpt_files, +( unsigned int id, std::list *hs_list, std::list::iterator *it, + std::mutex *hs_mtx, std::string path, ErrorList *el, std::unordered_set *all_wpt_files, WaypointQuadtree *all_waypoints, std::mutex *strtok_mtx, DatacheckEntryList *datacheckerrors ) -{ //std::cout << "Starting ReadWptThread " << id << std::endl; - while (hs_list->size()) +{ //printf("Starting ReadWptThread %02i\n", id); fflush(stdout); + while (*it != hs_list->end()) { hs_mtx->lock(); - if (!hs_list->size()) + if (*it == hs_list->end()) { hs_mtx->unlock(); return; } - //std::cout << "Thread " << id << " with hs_list->size()=" << hs_list->size() << std::endl; - HighwaySystem *h(hs_list->front()); - //std::cout << "Thread " << id << " assigned " << h->systemname << std::endl; - hs_list->pop_front(); - //std::cout << "Thread " << id << " hs_list->pop_front() successful." << std::endl; + HighwaySystem *h(**it); + //printf("ReadWptThread %02i assigned %s\n", id, h->systemname.data()); fflush(stdout); + (*it)++; + //printf("ReadWptThread %02i (*it)++\n", id); fflush(stdout); hs_mtx->unlock(); std::cout << h->systemname << std::flush; for (Route &r : h->route_list) diff --git a/siteupdate/cplusplus/threads/UserLogThread.cpp b/siteupdate/cplusplus/threads/UserLogThread.cpp index 7787c65f..76bc64f7 100644 --- a/siteupdate/cplusplus/threads/UserLogThread.cpp +++ b/siteupdate/cplusplus/threads/UserLogThread.cpp @@ -1,16 +1,19 @@ -void UserLogThread(unsigned int id, std::list *tl_list, std::mutex *mtx, ClinchedDBValues *clin_db_val, const double total_active_only_miles, const double total_active_preview_miles, std::list *highway_systems, std::string path) -{ //std::cout << "Starting UserLogThread " << id << std::endl; - while (tl_list->size()) +void UserLogThread +( unsigned int id, std::list *tl_list, std::list::iterator *it, + std::mutex *mtx, ClinchedDBValues *clin_db_val, const double total_active_only_miles, + const double total_active_preview_miles, std::list *highway_systems, std::string path +) +{ //printf("Starting UserLogThread %02i\n", id); fflush(stdout); + while (*it != tl_list->end()) { mtx->lock(); - if (!tl_list->size()) + if (*it == tl_list->end()) { mtx->unlock(); return; } - //std::cout << "Thread " << id << " with tl_list->size()=" << tl_list->size() << std::endl; - TravelerList *t(tl_list->front()); - //std::cout << "Thread " << id << " assigned " << t->traveler_name << std::endl; - tl_list->pop_front(); - //std::cout << "Thread " << id << " tl_list->pop_front() successful." << std::endl; + TravelerList *t(**it); + //printf("UserLogThread %02i assigned %s\n", id, t->traveler_name.data()); fflush(stdout); + (*it)++; + //printf("UserLogThread %02i (*it)++\n", id); fflush(stdout); mtx->unlock(); t->userlog(clin_db_val, total_active_only_miles, total_active_preview_miles, highway_systems, path); }