From 12d192e119f720ca349a7fd1467b882e73912712 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Tue, 23 Apr 2019 16:18:23 -0400 Subject: [PATCH 1/2] terminal LF in traveled graphs (C++) --- siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp b/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp index 1e3e0eb5..c2974a52 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp +++ b/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp @@ -479,6 +479,7 @@ class HighwayGraph // traveler names for (TravelerList *t : *traveler_lists) tmgfile << t->traveler_name << ' '; + tmgfile << '\n'; // sanity check on edges written if (num_traveled_edges != edge) @@ -552,6 +553,7 @@ class HighwayGraph // traveler names for (TravelerList *t : *traveler_lists) travelfile << t->traveler_name << ' '; + travelfile << '\n'; simplefile.close(); collapfile.close(); travelfile.close(); From 652efe3135ff6e65520a05c371d1a4dcce339c90 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Tue, 23 Apr 2019 19:23:21 -0400 Subject: [PATCH 2/2] C++: finalizing traveled graphs https://github.com/TravelMapping/DataProcessing/issues/199 https://github.com/TravelMapping/DataProcessing/pull/201 --- .../GraphGeneration/GraphListEntry.cpp | 2 +- .../classes/GraphGeneration/HGEdge.cpp | 2 +- .../classes/GraphGeneration/HighwayGraph.cpp | 34 ++++++++++++------ .../classes/HighwaySegment/HighwaySegment.cpp | 10 +++--- .../classes/HighwaySegment/HighwaySegment.h | 2 +- .../classes/TravelerList/TravelerList.cpp | 4 ++- .../cplusplus/functions/graph_generation.cpp | 36 +++++++++---------- siteupdate/cplusplus/siteupdate.cpp | 4 +-- .../cplusplus/threads/SubgraphThread.cpp | 6 ++-- 9 files changed, 58 insertions(+), 42 deletions(-) diff --git a/siteupdate/cplusplus/classes/GraphGeneration/GraphListEntry.cpp b/siteupdate/cplusplus/classes/GraphGeneration/GraphListEntry.cpp index 23188593..345d6097 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/GraphListEntry.cpp +++ b/siteupdate/cplusplus/classes/GraphGeneration/GraphListEntry.cpp @@ -33,7 +33,7 @@ class GraphListEntry std::string GraphListEntry::filename() { switch (form) { case 's': return root+"-simple.tmg"; - case 'c': return root+".tmg"; + case 'c': return root+"-collapsed.tmg"; case 't': return root+"-traveled.tmg"; default : return std::string("ERROR: GraphListEntry::filename() unexpected format token ('")+form+"')"; } diff --git a/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.cpp b/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.cpp index 04bb7209..fc3f4848 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.cpp +++ b/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.cpp @@ -235,7 +235,7 @@ std::string HGEdge::collapsed_tmg_line(std::list *systems, unsig // line appropriate for a tmg traveled edge file std::string HGEdge::traveled_tmg_line(std::list *systems, std::list *traveler_lists, unsigned int threadnum) { std::string line = std::to_string(vertex1->t_vertex_num[threadnum]) + " " + std::to_string(vertex2->t_vertex_num[threadnum]) + " " + label(systems); - line += " " + segment->clinchedby_code(traveler_lists); + line += " " + segment->clinchedby_code(traveler_lists, threadnum); char fstr[43]; for (HGVertex *intermediate : intermediate_points) { sprintf(fstr, " %.15g %.15g", intermediate->lat, intermediate->lng); diff --git a/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp b/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp index c2974a52..78868a3a 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp +++ b/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp @@ -331,10 +331,11 @@ class HighwayGraph return edge_set; } - std::unordered_set matching_traveled_edges(std::unordered_set &mv, GraphListEntry &g) + void matching_traveled_edges(std::unordered_set &mv, GraphListEntry &g, + std::unordered_set &edge_set, std::list &traveler_lists) { // return a set of edges for the traveled graph format, // optionally restricted by region or system or placeradius - std::unordered_set edge_set; + std::unordered_set trav_set; for (HGVertex *v : mv) { if (v->visibility < 1) continue; for (HGEdge *e : v->incident_t_edges) @@ -357,11 +358,15 @@ class HighwayGraph } if (sys_in_sys) system_match = 1; } - if (system_match) edge_set.insert(e); + if (system_match) + { edge_set.insert(e); + for (TravelerList *t : e->segment->clinched_by) trav_set.insert(t); + } } } } - return edge_set; + traveler_lists.assign(trav_set.begin(), trav_set.end()); + traveler_lists.sort(sort_travelers_by_name); } // write the entire set of highway data in .tmg format. @@ -454,7 +459,7 @@ class HighwayGraph { std::ofstream tmgfile(filename.data()); unsigned int num_traveled_edges = traveled_edge_count(); tmgfile << "TMG 2.0 traveled\n"; - tmgfile << num_traveled_vertices() << " " << num_traveled_edges << '\n'; + tmgfile << num_traveled_vertices() << ' ' << num_traveled_edges << ' ' << traveler_lists->size() << '\n'; // write visible vertices int t_vertex_num = 0; @@ -495,8 +500,7 @@ class HighwayGraph // restricted by regions in the list if given, // by systems in the list if given, // or to within a given area if placeradius is given - void write_subgraphs_tmg(std::vector &graph_vector, std::string path, size_t graphnum, - unsigned int threadnum, std::list *traveler_lists) + void write_subgraphs_tmg(std::vector &graph_vector, std::string path, size_t graphnum, unsigned int threadnum) { unsigned int cv_count, tv_count; std::ofstream simplefile((path+graph_vector[graphnum].filename()).data()); std::ofstream collapfile((path+graph_vector[graphnum+1].filename()).data()); @@ -504,7 +508,15 @@ class HighwayGraph std::unordered_set mv = matching_vertices(graph_vector[graphnum], cv_count, tv_count); std::unordered_set mse = matching_simple_edges(mv, graph_vector[graphnum]); std::unordered_set mce = matching_collapsed_edges(mv, graph_vector[graphnum]); - std::unordered_set mte = matching_traveled_edges(mv, graph_vector[graphnum]); + std::unordered_set mte; + std::list traveler_lists; + matching_traveled_edges(mv, graph_vector[graphnum], mte, traveler_lists); + // assign traveler numbers + unsigned int travnum = 0; + for (TravelerList *t : traveler_lists) + { t->traveler_num[threadnum] = travnum; + travnum++; + } std::cout << graph_vector[graphnum].tag() << '(' << mv.size() << ',' << mse.size() << ") " << '(' << cv_count << ',' << mce.size() << ") " @@ -514,7 +526,7 @@ class HighwayGraph travelfile << "TMG 2.0 traveled\n"; simplefile << mv.size() << ' ' << mse.size() << '\n'; collapfile << cv_count << ' ' << mce.size() << '\n'; - travelfile << tv_count << ' ' << mte.size() << '\n'; + travelfile << tv_count << ' ' << mte.size() << ' ' << traveler_lists.size() << '\n'; // write vertices unsigned int sv = 0; @@ -549,9 +561,9 @@ class HighwayGraph for (HGEdge *e : mce) collapfile << e->collapsed_tmg_line(graph_vector[graphnum].systems, threadnum) << '\n'; for (HGEdge *e : mte) - travelfile << e->traveled_tmg_line(graph_vector[graphnum].systems, traveler_lists, threadnum) << '\n'; + travelfile << e->traveled_tmg_line(graph_vector[graphnum].systems, &traveler_lists, threadnum) << '\n'; // traveler names - for (TravelerList *t : *traveler_lists) + for (TravelerList *t : traveler_lists) travelfile << t->traveler_name << ' '; travelfile << '\n'; simplefile.close(); diff --git a/siteupdate/cplusplus/classes/HighwaySegment/HighwaySegment.cpp b/siteupdate/cplusplus/classes/HighwaySegment/HighwaySegment.cpp index 2020cc52..22cd39d8 100644 --- a/siteupdate/cplusplus/classes/HighwaySegment/HighwaySegment.cpp +++ b/siteupdate/cplusplus/classes/HighwaySegment/HighwaySegment.cpp @@ -70,22 +70,24 @@ unsigned int HighwaySegment::index() return ""; }//*/ -std::string HighwaySegment::clinchedby_code(std::list *traveler_lists) +std::string HighwaySegment::clinchedby_code(std::list *traveler_lists, unsigned int threadnum) { // Return a hexadecimal string encoding which travelers have clinched this segment, for use in "traveled" graph files // Each character stores info for traveler #n thru traveler #n+3 // The first character stores traveler 0 thru traveler 3, // The second character stores traveler 4 thru traveler 7, etc. // For each character, the low-order bit stores traveler n, and the high bit traveler n+3. + + if (traveler_lists->empty()) return "0"; std::string code(ceil(double(traveler_lists->size())/4), '0'); //std::cout << str() << " code string initialized (" << clinched_by.size() << '/' << traveler_lists->size() << ')' << std::endl; //unsigned int num = 0; for (TravelerList* t : clinched_by) - { //std::cout << "\t" << num << ": TravNum = " << t->traveler_num << ": " << t->traveler_name << std::endl; - //std::cout << "\t" << num << ": TravNum/4 = " << t->traveler_num/4 << std::endl; + { //std::cout << "\t" << num << ": TravNum = " << t->traveler_num[threadnum] << ": " << t->traveler_name << std::endl; + //std::cout << "\t" << num << ": TravNum/4 = " << t->traveler_num[threadnum]/4 << std::endl; //std::cout << "\t" << num << ": TravNum%4 = " << TravNum%4 << std::endl; //std::cout << "\t" << num << ": 2 ^ TravNum%4 = " << pow(2, TravNum%4) << std::endl; //std::cout << "\t" << num << ": code[" << TravNum/4 << "] += int(" << pow(2, TravNum%4) << ")" << std::endl; - code[t->traveler_num/4] += int(pow(2, t->traveler_num%4)); + code[t->traveler_num[threadnum]/4] += int(pow(2, t->traveler_num[threadnum]%4)); //num++; } //std::cout << "travelers written to array" << std::endl; diff --git a/siteupdate/cplusplus/classes/HighwaySegment/HighwaySegment.h b/siteupdate/cplusplus/classes/HighwaySegment/HighwaySegment.h index 6b3c0176..234fb556 100644 --- a/siteupdate/cplusplus/classes/HighwaySegment/HighwaySegment.h +++ b/siteupdate/cplusplus/classes/HighwaySegment/HighwaySegment.h @@ -19,6 +19,6 @@ class HighwaySegment std::string segment_name(); unsigned int index(); //std::string concurrent_travelers_sanity_check(); - std::string clinchedby_code(std::list *); + std::string clinchedby_code(std::list *, unsigned int); void compute_stats(); }; diff --git a/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp b/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp index 690c30eb..a1e65ba4 100644 --- a/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp +++ b/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp @@ -23,7 +23,7 @@ class TravelerList std::unordered_map routes_traveled; // mileage per traveled route std::unordered_map con_routes_clinched; // clinch count per system //std::unordered_map routes_clinched; // commented out in original siteupdate.py - unsigned int traveler_num; + unsigned int *traveler_num; unsigned int active_systems_traveled; unsigned int active_systems_clinched; unsigned int preview_systems_traveled; @@ -35,6 +35,8 @@ class TravelerList active_systems_clinched = 0; preview_systems_traveled = 0; preview_systems_clinched = 0; + traveler_num = new unsigned int[args->numthreads]; + // deleted on termination of program traveler_name = travname.substr(0, travname.size()-5); // strip ".list" from end of travname std::string filename = args->logfilepath+"/users/"+traveler_name+".log"; std::ofstream log(filename.data()); diff --git a/siteupdate/cplusplus/functions/graph_generation.cpp b/siteupdate/cplusplus/functions/graph_generation.cpp index 750f33d4..34070999 100644 --- a/siteupdate/cplusplus/functions/graph_generation.cpp +++ b/siteupdate/cplusplus/functions/graph_generation.cpp @@ -31,16 +31,16 @@ else { list *regions; #endif//*/ { cout << et.et() << "Writing master TM simple graph file, tm-master-simple.tmg" << endl; graph_data.write_master_tmg_simple(&graph_vector[0], args.graphfilepath+"/tm-master-simple.tmg"); - cout << et.et() << "Writing master TM collapsed graph file, tm-master.tmg." << endl; - graph_data.write_master_tmg_collapsed(&graph_vector[1], args.graphfilepath+"/tm-master.tmg", 0); + cout << et.et() << "Writing master TM collapsed graph file, tm-master-collapsed.tmg." << endl; + graph_data.write_master_tmg_collapsed(&graph_vector[1], args.graphfilepath+"/tm-master-collapsed.tmg", 0); cout << et.et() << "Writing master TM traveled graph file, tm-master.tmg." << endl; graph_data.write_master_tmg_traveled(&graph_vector[2], args.graphfilepath+"/tm-master-traveled.tmg", &traveler_lists, 0); } /*#ifdef threading_enabled else { cout << et.et() << "Writing master TM simple graph file, tm-master-simple.tmg" << endl; thr[0] = new thread(MasterTmgSimpleThread, &graph_data, &graph_vector[0], args.graphfilepath+"/tm-master-simple.tmg"); - cout << et.et() << "Writing master TM collapsed graph file, tm-master.tmg." << endl; - thr[1] = new thread(MasterTmgCollapsedThread, &graph_data, &graph_vector[1], args.graphfilepath+"/tm-master.tmg"); + cout << et.et() << "Writing master TM collapsed graph file, tm-master-collapsed.tmg." << endl; + thr[1] = new thread(MasterTmgCollapsedThread, &graph_data, &graph_vector[1], args.graphfilepath+"/tm-master-collapsed.tmg"); thr[0]->join(); thr[1]->join(); delete thr[0]; @@ -86,14 +86,14 @@ else { list *regions; #ifdef threading_enabled // set up for threaded subgraph generation for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &traveler_lists); + thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/"); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); for (unsigned int t = 0; t < args.numthreads; t++) delete thr[t]; #else while (graphnum < graph_vector.size()) - { graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &traveler_lists); + { graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0); graphnum += 3; } #endif @@ -125,14 +125,14 @@ else { list *regions; #ifdef threading_enabled // set up for threaded subgraph generation for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &traveler_lists); + thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/"); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); for (unsigned int t = 0; t < args.numthreads; t++) delete thr[t]; #else while (graphnum < graph_vector.size()) - { graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &traveler_lists); + { graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0); graphnum += 3; } #endif @@ -174,14 +174,14 @@ else { list *regions; #ifdef threading_enabled // set up for threaded subgraph generation for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &traveler_lists); + thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/"); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); for (unsigned int t = 0; t < args.numthreads; t++) delete thr[t]; #else while (graphnum < graph_vector.size()) - { graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &traveler_lists); + { graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0); graphnum += 3; } #endif @@ -226,14 +226,14 @@ else { list *regions; #ifdef threading_enabled // set up for threaded subgraph generation for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &traveler_lists); + thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/"); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); for (unsigned int t = 0; t < args.numthreads; t++) delete thr[t]; #else while (graphnum < graph_vector.size()) - { graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &traveler_lists); + { graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0); graphnum += 3; } #endif @@ -277,14 +277,14 @@ else { list *regions; #ifdef threading_enabled // set up for threaded subgraph generation for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &traveler_lists); + thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/"); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); for (unsigned int t = 0; t < args.numthreads; t++) delete thr[t]; #else while (graphnum < graph_vector.size()) - { graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &traveler_lists); + { graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0); graphnum += 3; } #endif @@ -319,14 +319,14 @@ else { list *regions; #ifdef threading_enabled // set up for threaded subgraph generation for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &traveler_lists); + thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/"); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); for (unsigned int t = 0; t < args.numthreads; t++) delete thr[t]; #else while (graphnum < graph_vector.size()) - { graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &traveler_lists); + { graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0); graphnum += 3; } #endif @@ -361,14 +361,14 @@ else { list *regions; #ifdef threading_enabled // set up for threaded subgraph generation for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &traveler_lists); + thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/"); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); for (unsigned int t = 0; t < args.numthreads; t++) delete thr[t]; #else while (graphnum < graph_vector.size()) - { graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0, &traveler_lists); + { graph_data.write_subgraphs_tmg(graph_vector, args.graphfilepath + "/", graphnum, 0); graphnum += 3; } #endif diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index fc5ec386..80455530 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -416,10 +416,10 @@ int main(int argc, char *argv[]) #endif cout << " processed " << traveler_lists.size() << " traveler list files." << endl; traveler_lists.sort(sort_travelers_by_name); - // assign traveler numbers + // assign traveler numbers for master traveled graph unsigned int travnum = 0; for (TravelerList *t : traveler_lists) - { t->traveler_num = travnum; + { t->traveler_num[0] = travnum; travnum++; } diff --git a/siteupdate/cplusplus/threads/SubgraphThread.cpp b/siteupdate/cplusplus/threads/SubgraphThread.cpp index d56e96f2..653fd0c7 100644 --- a/siteupdate/cplusplus/threads/SubgraphThread.cpp +++ b/siteupdate/cplusplus/threads/SubgraphThread.cpp @@ -1,5 +1,5 @@ -void SubgraphThread(unsigned int id, HighwayGraph *graph_data, std::vector *graph_vector, -size_t *index, std::mutex *mtx, std::string path, std::list *traveler_lists) +void SubgraphThread(unsigned int id, HighwayGraph *graph_data, +std::vector *graph_vector, size_t *index,std::mutex *mtx, std::string path) { //std::cout << "Starting SubgraphThread " << id << std::endl; while (*index < graph_vector->size()) { mtx->lock(); @@ -12,6 +12,6 @@ size_t *index, std::mutex *mtx, std::string path, std::list *trav size_t i = *index; *index += 3; mtx->unlock(); - graph_data->write_subgraphs_tmg(*graph_vector, path, i, id, traveler_lists); + graph_data->write_subgraphs_tmg(*graph_vector, path, i, id); } }