From bc964258e733abafe6ff2550f16fa12788a51439 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Mon, 24 Jun 2019 18:05:10 -0400 Subject: [PATCH 1/3] sql file concurrent with graph setup --- .../classes/GraphGeneration/HighwayGraph.cpp | 6 +- siteupdate/cplusplus/functions/sql_file.cpp | 175 ++++++++++++------ siteupdate/cplusplus/siteupdate.cpp | 26 ++- 3 files changed, 142 insertions(+), 65 deletions(-) diff --git a/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp b/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp index 7bed22e8..8d15c2a2 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp +++ b/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp @@ -30,7 +30,7 @@ class HighwayGraph // or preview system, or is colocated and not at the front // of its colocation list unsigned int counter = 0; - std::cout << et.et() << "Creating unique names and vertices" << std::flush; + std::cout << et.et() + "Creating unique names and vertices" << std::flush; for (Waypoint *w : all_waypoints.point_list()) { if (counter % 10000 == 0) std::cout << '.' << std::flush; counter++; @@ -80,7 +80,7 @@ class HighwayGraph // create edges counter = 0; - std::cout << et.et() << "Creating edges" << std::flush; + std::cout << et.et() + "Creating edges" << std::flush; for (HighwaySystem *h : highway_systems) { if (h->devel()) continue; if (counter % 6 == 0) std::cout << '.' << std::flush; @@ -95,7 +95,7 @@ class HighwayGraph // compress edges adjacent to hidden vertices counter = 0; - std::cout << et.et() << "Compressing collapsed edges" << std::flush; + std::cout << et.et() + "Compressing collapsed edges" << std::flush; for (std::pair wv : vertices) { if (counter % 10000 == 0) std::cout << '.' << std::flush; counter++; diff --git a/siteupdate/cplusplus/functions/sql_file.cpp b/siteupdate/cplusplus/functions/sql_file.cpp index dc528480..0542639e 100644 --- a/siteupdate/cplusplus/functions/sql_file.cpp +++ b/siteupdate/cplusplus/functions/sql_file.cpp @@ -1,10 +1,18 @@ -if (args.errorcheck) - cout << et.et() << "SKIPPING database file." << endl; -else { cout << et.et() << "Writing database file " << args.databasename << ".sql." << endl; +void sqlfile1 + ( ElapsedTime *et, + Arguments *args, + std::list *all_regions, + std::list> *continents, + std::list> *countries, + std::list *highway_systems, + std::list *traveler_lists, + ClinchedDBValues *clin_db_val, + std::list> *updates, + std::list> *systemupdates + ){ // Once all data is read in and processed, create a .sql file that will // create all of the DB tables to be used by other parts of the project - filename = args.databasename+".sql"; - ofstream sqlfile(filename.data()); + std::ofstream sqlfile((args->databasename+".sql").data()); // Note: removed "USE" line, DB name must be specified on the mysql command line // we have to drop tables in the right order to avoid foreign key errors @@ -29,11 +37,13 @@ else { cout << et.et() << "Writing database file " << args.databasename << ".sql sqlfile << "DROP TABLE IF EXISTS continents;\n"; // first, continents, countries, and regions - cout << et.et() << "...continents, countries, regions" << endl; + #ifndef threading_enabled + std::cout << et->et() << "...continents, countries, regions" << std::endl; + #endif sqlfile << "CREATE TABLE continents (code VARCHAR(3), name VARCHAR(15), PRIMARY KEY(code));\n"; sqlfile << "INSERT INTO continents VALUES\n"; bool first = 1; - for (pair c : continents) + for (std::pair c : *continents) { if (!first) sqlfile << ','; first = 0; sqlfile << "('" << c.first << "','" << c.second << "')\n"; @@ -43,7 +53,7 @@ else { cout << et.et() << "Writing database file " << args.databasename << ".sql sqlfile << "CREATE TABLE countries (code VARCHAR(3), name VARCHAR(32), PRIMARY KEY(code));\n"; sqlfile << "INSERT INTO countries VALUES\n"; first = 1; - for (pair c : countries) + for (std::pair c : *countries) { if (!first) sqlfile << ','; first = 0; sqlfile << "('" << c.first << "','" << double_quotes(c.second) << "')\n"; @@ -54,7 +64,7 @@ else { cout << et.et() << "Writing database file " << args.databasename << ".sql sqlfile << "PRIMARY KEY(code), FOREIGN KEY (country) REFERENCES countries(code), FOREIGN KEY (continent) REFERENCES continents(code));\n"; sqlfile << "INSERT INTO regions VALUES\n"; first = 1; - for (Region &r : all_regions) + for (Region &r : *all_regions) { if (!first) sqlfile << ','; first = 0; sqlfile << "('" << r.code << "','" + double_quotes(r.name) << "','" << r.country_code() + "','" + r.continent_code() + "','" << r.type << "')\n"; @@ -66,13 +76,15 @@ else { cout << et.et() << "Writing database file " << args.databasename << ".sql // color for its mapping, a level (one of active, preview, devel), and // a boolean indicating if the system is active for mapping in the // project in the field 'active' - cout << et.et() << "...systems" << endl; + #ifndef threading_enabled + std::cout << et->et() << "...systems" << std::endl; + #endif sqlfile << "CREATE TABLE systems (systemName VARCHAR(10), countryCode CHAR(3), fullName VARCHAR(60), color "; sqlfile << "VARCHAR(16), level VARCHAR(10), tier INTEGER, csvOrder INTEGER, PRIMARY KEY(systemName));\n"; sqlfile << "INSERT INTO systems VALUES\n"; first = 1; unsigned int csvOrder = 0; - for (HighwaySystem *h : highway_systems) + for (HighwaySystem *h : *highway_systems) { if (!first) sqlfile << ','; first = 0; sqlfile << "('" << h->systemname << "','" << h->country->first << "','" @@ -83,13 +95,15 @@ else { cout << et.et() << "Writing database file " << args.databasename << ".sql sqlfile << ";\n"; // next, a table of highways, with the same fields as in the first line - cout << et.et() << "...routes" << endl; + #ifndef threading_enabled + std::cout << et->et() << "...routes" << std::endl; + #endif sqlfile << "CREATE TABLE routes (systemName VARCHAR(10), region VARCHAR(8), route VARCHAR(16), banner VARCHAR(6), abbrev VARCHAR(3), city VARCHAR(100), root "; sqlfile << "VARCHAR(32), mileage FLOAT, rootOrder INTEGER, csvOrder INTEGER, PRIMARY KEY(root), FOREIGN KEY (systemName) REFERENCES systems(systemName));\n"; sqlfile << "INSERT INTO routes VALUES\n"; first = 1; csvOrder = 0; - for (HighwaySystem *h : highway_systems) + for (HighwaySystem *h : *highway_systems) for (Route &r : h->route_list) { if (!first) sqlfile << ','; first = 0; @@ -99,13 +113,15 @@ else { cout << et.et() << "Writing database file " << args.databasename << ".sql sqlfile << ";\n"; // connected routes table, but only first "root" in each in this table - cout << et.et() << "...connectedRoutes" << endl; + #ifndef threading_enabled + std::cout << et->et() << "...connectedRoutes" << std::endl; + #endif sqlfile << "CREATE TABLE connectedRoutes (systemName VARCHAR(10), route VARCHAR(16), banner VARCHAR(6), groupName VARCHAR(100), firstRoot "; sqlfile << "VARCHAR(32), mileage FLOAT, csvOrder INTEGER, PRIMARY KEY(firstRoot), FOREIGN KEY (firstRoot) REFERENCES routes(root));\n"; sqlfile << "INSERT INTO connectedRoutes VALUES\n"; first = 1; csvOrder = 0; - for (HighwaySystem *h : highway_systems) + for (HighwaySystem *h : *highway_systems) for (ConnectedRoute &cr : h->con_route_list) { if (!first) sqlfile << ','; first = 0; @@ -116,10 +132,12 @@ else { cout << et.et() << "Writing database file " << args.databasename << ".sql // This table has remaining roots for any connected route // that connects multiple routes/roots - cout << et.et() << "...connectedRouteRoots" << endl; + #ifndef threading_enabled + std::cout << et->et() << "...connectedRouteRoots" << std::endl; + #endif sqlfile << "CREATE TABLE connectedRouteRoots (firstRoot VARCHAR(32), root VARCHAR(32), FOREIGN KEY (firstRoot) REFERENCES connectedRoutes(firstRoot));\n"; first = 1; - for (HighwaySystem *h : highway_systems) + for (HighwaySystem *h : *highway_systems) for (ConnectedRoute &cr : h->con_route_list) for (unsigned int i = 1; i < cr.roots.size(); i++) { if (first) sqlfile << "INSERT INTO connectedRouteRoots VALUES\n"; @@ -130,11 +148,13 @@ else { cout << et.et() << "Writing database file " << args.databasename << ".sql sqlfile << ";\n"; // Now, a table with raw highway route data: list of points, in order, that define the route - cout << et.et() << "...waypoints" << endl; + #ifndef threading_enabled + std::cout << et->et() << "...waypoints" << std::endl; + #endif sqlfile << "CREATE TABLE waypoints (pointId INTEGER, pointName VARCHAR(20), latitude DOUBLE, longitude DOUBLE, "; sqlfile << "root VARCHAR(32), PRIMARY KEY(pointId), FOREIGN KEY (root) REFERENCES routes(root));\n"; unsigned int point_num = 0; - for (HighwaySystem *h : highway_systems) + for (HighwaySystem *h : *highway_systems) for (Route &r : h->route_list) { sqlfile << "INSERT INTO waypoints VALUES\n"; first = 1; @@ -153,12 +173,14 @@ else { cout << et.et() << "Writing database file " << args.databasename << ".sql sqlfile << "CREATE INDEX `longitude` ON waypoints(`longitude`);\n"; // Table of all HighwaySegments. - cout << et.et() << "...segments" << endl; + #ifndef threading_enabled + std::cout << et->et() << "...segments" << std::endl; + #endif sqlfile << "CREATE TABLE segments (segmentId INTEGER, waypoint1 INTEGER, waypoint2 INTEGER, root VARCHAR(32), PRIMARY KEY (segmentId), FOREIGN KEY (waypoint1) "; sqlfile << "REFERENCES waypoints(pointId), FOREIGN KEY (waypoint2) REFERENCES waypoints(pointId), FOREIGN KEY (root) REFERENCES routes(root));\n"; unsigned int segment_num = 0; - vector clinched_list; - for (HighwaySystem *h : highway_systems) + std::vector clinched_list; + for (HighwaySystem *h : *highway_systems) for (Route &r : h->route_list) { sqlfile << "INSERT INTO segments VALUES\n"; first = 1; @@ -167,7 +189,7 @@ else { cout << et.et() << "Writing database file " << args.databasename << ".sql first = 0; sqlfile << '(' << s->csv_line(segment_num) << ")\n"; for (TravelerList *t : s->clinched_by) - clinched_list.push_back("'" + to_string(segment_num) + "','" + t->traveler_name + "'"); + clinched_list.push_back("'" + std::to_string(segment_num) + "','" + t->traveler_name + "'"); segment_num += 1; } sqlfile << ";\n"; @@ -175,7 +197,9 @@ else { cout << et.et() << "Writing database file " << args.databasename << ".sql // maybe a separate traveler table will make sense but for now, I'll just use // the name from the .list name - cout << et.et() << "...clinched" << endl; + #ifndef threading_enabled + std::cout << et->et() << "...clinched" << std::endl; + #endif sqlfile << "CREATE TABLE clinched (segmentId INTEGER, traveler VARCHAR(48), FOREIGN KEY (segmentId) REFERENCES segments(segmentId));\n"; for (size_t start = 0; start < clinched_list.size(); start += 10000) { sqlfile << "INSERT INTO clinched VALUES\n"; @@ -190,11 +214,13 @@ else { cout << et.et() << "Writing database file " << args.databasename << ".sql // overall mileage by region data (with concurrencies accounted for, // active systems only then active+preview) - cout << et.et() << "...overallMileageByRegion" << endl; + #ifndef threading_enabled + std::cout << et->et() << "...overallMileageByRegion" << std::endl; + #endif sqlfile << "CREATE TABLE overallMileageByRegion (region VARCHAR(8), activeMileage FLOAT, activePreviewMileage FLOAT);\n"; sqlfile << "INSERT INTO overallMileageByRegion VALUES\n"; first = 1; - for (Region ®ion : all_regions) + for (Region ®ion : *all_regions) { if (region.active_only_mileage+region.active_preview_mileage == 0) continue; if (!first) sqlfile << ','; first = 0; @@ -206,14 +232,16 @@ else { cout << et.et() << "Writing database file " << args.databasename << ".sql // system mileage by region data (with concurrencies accounted for, // active systems and preview systems only) - cout << et.et() << "...systemMileageByRegion" << endl; + #ifndef threading_enabled + std::cout << et->et() << "...systemMileageByRegion" << std::endl; + #endif sqlfile << "CREATE TABLE systemMileageByRegion (systemName VARCHAR(10), region VARCHAR(8), "; sqlfile << "mileage FLOAT, FOREIGN KEY (systemName) REFERENCES systems(systemName));\n"; sqlfile << "INSERT INTO systemMileageByRegion VALUES\n"; first = 1; - for (HighwaySystem *h : highway_systems) + for (HighwaySystem *h : *highway_systems) if (h->active_or_preview()) - for (pair rm : h->mileage_by_region) + for (std::pair rm : h->mileage_by_region) { if (!first) sqlfile << ','; first = 0; char fstr[35]; @@ -224,12 +252,14 @@ else { cout << et.et() << "Writing database file " << args.databasename << ".sql // clinched overall mileage by region data (with concurrencies // accounted for, active systems and preview systems only) - cout << et.et() << "...clinchedOverallMileageByRegion" << endl; + #ifndef threading_enabled + std::cout << et->et() << "...clinchedOverallMileageByRegion" << std::endl; + #endif sqlfile << "CREATE TABLE clinchedOverallMileageByRegion (region VARCHAR(8), traveler VARCHAR(48), activeMileage FLOAT, activePreviewMileage FLOAT);\n"; sqlfile << "INSERT INTO clinchedOverallMileageByRegion VALUES\n"; first = 1; - for (TravelerList *t : traveler_lists) - for (pair rm : t->active_preview_mileage_by_region) + for (TravelerList *t : *traveler_lists) + for (std::pair rm : t->active_preview_mileage_by_region) { if (!first) sqlfile << ','; first = 0; double active_miles = 0; @@ -243,12 +273,14 @@ else { cout << et.et() << "Writing database file " << args.databasename << ".sql // clinched system mileage by region data (with concurrencies accounted // for, active systems and preview systems only) - cout << et.et() << "...clinchedSystemMileageByRegion" << endl; + #ifndef threading_enabled + std::cout << et->et() << "...clinchedSystemMileageByRegion" << std::endl; + #endif sqlfile << "CREATE TABLE clinchedSystemMileageByRegion (systemName VARCHAR(10), region VARCHAR(8), traveler "; sqlfile << "VARCHAR(48), mileage FLOAT, FOREIGN KEY (systemName) REFERENCES systems(systemName));\n"; sqlfile << "INSERT INTO clinchedSystemMileageByRegion VALUES\n"; first = 1; - for (string &line : clin_db_val.csmbr_values) + for (std::string &line : clin_db_val->csmbr_values) { if (!first) sqlfile << ','; first = 0; sqlfile << line << '\n'; @@ -257,40 +289,46 @@ else { cout << et.et() << "Writing database file " << args.databasename << ".sql // clinched mileage by connected route, active systems and preview // systems only - cout << et.et() << "...clinchedConnectedRoutes" << endl; + #ifndef threading_enabled + std::cout << et->et() << "...clinchedConnectedRoutes" << std::endl; + #endif sqlfile << "CREATE TABLE clinchedConnectedRoutes (route VARCHAR(32), traveler VARCHAR(48), mileage "; sqlfile << "FLOAT, clinched BOOLEAN, FOREIGN KEY (route) REFERENCES connectedRoutes(firstRoot));\n"; - for (size_t start = 0; start < clin_db_val.ccr_values.size(); start += 10000) + for (size_t start = 0; start < clin_db_val->ccr_values.size(); start += 10000) { sqlfile << "INSERT INTO clinchedConnectedRoutes VALUES\n"; first = 1; - for (size_t line = start; line < start+10000 && line < clin_db_val.ccr_values.size(); line++) + for (size_t line = start; line < start+10000 && line < clin_db_val->ccr_values.size(); line++) { if (!first) sqlfile << ','; first = 0; - sqlfile << clin_db_val.ccr_values[line] << '\n'; + sqlfile << clin_db_val->ccr_values[line] << '\n'; } sqlfile << ";\n"; } // clinched mileage by route, active systems and preview systems only - cout << et.et() << "...clinchedRoutes" << endl; + #ifndef threading_enabled + std::cout << et->et() << "...clinchedRoutes" << std::endl; + #endif sqlfile << "CREATE TABLE clinchedRoutes (route VARCHAR(32), traveler VARCHAR(48), mileage FLOAT, clinched BOOLEAN, FOREIGN KEY (route) REFERENCES routes(root));\n"; - for (size_t start = 0; start < clin_db_val.cr_values.size(); start += 10000) + for (size_t start = 0; start < clin_db_val->cr_values.size(); start += 10000) { sqlfile << "INSERT INTO clinchedRoutes VALUES\n"; first = 1; - for (size_t line = start; line < start+10000 && line < clin_db_val.cr_values.size(); line++) + for (size_t line = start; line < start+10000 && line < clin_db_val->cr_values.size(); line++) { if (!first) sqlfile << ','; first = 0; - sqlfile << clin_db_val.cr_values[line] << '\n'; + sqlfile << clin_db_val->cr_values[line] << '\n'; } sqlfile << ";\n"; } // updates entries - cout << et.et() << "...updates" << endl; + #ifndef threading_enabled + std::cout << et->et() << "...updates" << std::endl; + #endif sqlfile << "CREATE TABLE updates (date VARCHAR(10), region VARCHAR(60), route VARCHAR(80), root VARCHAR(32), description VARCHAR(1024));\n"; sqlfile << "INSERT INTO updates VALUES\n"; first = 1; - for (array &update : updates) + for (std::array &update : *updates) { if (!first) sqlfile << ','; first = 0; sqlfile << "('" << update[0] << "','" << double_quotes(update[1]) << "','" << double_quotes(update[2]) @@ -299,25 +337,41 @@ else { cout << et.et() << "Writing database file " << args.databasename << ".sql sqlfile << ";\n"; // systemUpdates entries - cout << et.et() << "...systemUpdates" << endl; + #ifndef threading_enabled + std::cout << et->et() << "...systemUpdates" << std::endl; + #endif sqlfile << "CREATE TABLE systemUpdates (date VARCHAR(10), region VARCHAR(48), systemName VARCHAR(10), description VARCHAR(128), statusChange VARCHAR(16));\n"; sqlfile << "INSERT INTO systemUpdates VALUES\n"; first = 1; - for (array &systemupdate : systemupdates) + for (std::array &systemupdate : *systemupdates) { if (!first) sqlfile << ','; first = 0; sqlfile << "('" << systemupdate[0] << "','" << double_quotes(systemupdate[1]) << "','" << systemupdate[2] << "','" << double_quotes(systemupdate[3]) << "','" << systemupdate[4] << "')\n"; } sqlfile << ";\n"; + sqlfile.close(); + std::cout << '\n' + et->et() + "Pause writing database file " + args->databasename + ".sql.\n" << std::flush; + } + +void sqlfile2 + ( ElapsedTime *et, + Arguments *args, + std::list> *graph_types, + std::vector *graph_vector, + DatacheckEntryList *datacheckerrors + ) + { std::ofstream sqlfile((args->databasename+".sql").data(), std::ios::app); // datacheck errors into the db - cout << et.et() << "...datacheckErrors" << endl; + #ifndef threading_enabled + std::cout << et->et() << "...datacheckErrors" << std::endl; + #endif sqlfile << "CREATE TABLE datacheckErrors (route VARCHAR(32), label1 VARCHAR(50), label2 VARCHAR(20), label3 VARCHAR(20), "; sqlfile << "code VARCHAR(20), value VARCHAR(32), falsePositive BOOLEAN, FOREIGN KEY (route) REFERENCES routes(root));\n"; if (datacheckerrors->entries.size()) { sqlfile << "INSERT INTO datacheckErrors VALUES\n"; - first = 1; + bool first = 1; for (DatacheckEntry &d : datacheckerrors->entries) { if (!first) sqlfile << ','; first = 0; @@ -329,15 +383,18 @@ else { cout << et.et() << "Writing database file " << args.databasename << ".sql sqlfile << ";\n"; // update graph info in DB if graphs were generated - if (!args.skipgraphs) - { cout << et.et() << "...graphs" << endl; + if (!args->skipgraphs) + { + #ifndef threading_enabled + std::cout << et->et() << "...graphs" << std::endl; + #endif sqlfile << "DROP TABLE IF EXISTS graphs;\n"; sqlfile << "DROP TABLE IF EXISTS graphTypes;\n"; sqlfile << "CREATE TABLE graphTypes (category VARCHAR(12), descr VARCHAR(100), longDescr TEXT, PRIMARY KEY(category));\n"; - if (graph_types.size()) + if (graph_types->size()) { sqlfile << "INSERT INTO graphTypes VALUES\n"; - first = 1; - for (array &g : graph_types) + bool first = 1; + for (std::array &g : *graph_types) { if (!first) sqlfile << ','; first = 0; sqlfile << "('" << g[0] << "','" << g[1] << "','" << g[2] << "')\n"; @@ -346,14 +403,14 @@ else { cout << et.et() << "Writing database file " << args.databasename << ".sql } sqlfile << "CREATE TABLE graphs (filename VARCHAR(32), descr VARCHAR(100), vertices INTEGER, edges INTEGER, travelers INTEGER, "; sqlfile << "format VARCHAR(10), category VARCHAR(12), FOREIGN KEY (category) REFERENCES graphTypes(category));\n"; - if (graph_vector.size()) + if (graph_vector->size()) { sqlfile << "INSERT INTO graphs VALUES\n"; - for (size_t g = 0; g < graph_vector.size(); g++) + for (size_t g = 0; g < graph_vector->size(); g++) { if (g) sqlfile << ','; - sqlfile << "('" << graph_vector[g].filename() << "','" << double_quotes(graph_vector[g].descr) - << "','" << graph_vector[g].vertices << "','" << graph_vector[g].edges - << "','" << graph_vector[g].travelers << "','" << graph_vector[g].format() - << "','" << graph_vector[g].category() << "')\n"; + sqlfile << "('" << graph_vector->at(g).filename() << "','" << double_quotes(graph_vector->at(g).descr) + << "','" << graph_vector->at(g).vertices << "','" << graph_vector->at(g).edges + << "','" << graph_vector->at(g).travelers << "','" << graph_vector->at(g).format() + << "','" << graph_vector->at(g).category() << "')\n"; } sqlfile << ";\n"; } diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index 6c3921a0..a54b306b 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -83,6 +83,7 @@ class HGEdge; #include "threads/UserLogThread.cpp" #include "threads/SubgraphThread.cpp" #include "threads/MasterTmgThreads.cpp" +#include "functions/sql_file.cpp" using namespace std; int main(int argc, char *argv[]) @@ -223,7 +224,7 @@ int main(int argc, char *argv[]) { cout << et.et() << "Checking for duplicate list names in routes, roots in routes and connected routes." << endl; unordered_set roots, list_names, duplicate_list_names; unordered_set con_roots; - + for (HighwaySystem *h : highway_systems) { for (Route r : h->route_list) { if (roots.find(r.root) == roots.end()) roots.insert(r.root); @@ -677,7 +678,7 @@ int main(int argc, char *argv[]) list region_entries; for (Region region : all_regions) if (region.overall_mileage) - { sprintf(fstr, ": %.2f (active), %.2f (active, preview) %.2f (active, preview, devel)\n", + { sprintf(fstr, ": %.2f (active), %.2f (active, preview) %.2f (active, preview, devel)\n", region.active_only_mileage, region.active_preview_mileage, region.overall_mileage); region_entries.push_back(region.code + fstr); } @@ -943,6 +944,14 @@ int main(int argc, char *argv[]) return 0; } + #ifdef threading_enabled + std::cout << et.et() << "Start writing database file " << args.databasename << ".sql.\n" << std::flush; + thread sqlthread; + if (!args.errorcheck) + { sqlthread=thread(sqlfile1, &et, &args, &all_regions, &continents, &countries, &highway_systems, &traveler_lists, &clin_db_val, &updates, &systemupdates); + } + #endif + #include "functions/graph_generation.cpp" // now mark false positives @@ -1002,7 +1011,18 @@ int main(int argc, char *argv[]) if (!d.fp) logfile << d.str() << '\n'; logfile.close(); - #include "functions/sql_file.cpp" + if (args.errorcheck) + cout << et.et() << "SKIPPING database file." << endl; + else { + #ifdef threading_enabled + sqlthread.join(); + std::cout << et.et() << "Resume writing database file " << args.databasename << ".sql.\n" << std::flush; + #else + std::cout << et.et() << "Writing database file " << args.databasename << ".sql.\n" << std::flush; + sqlfile1(&et, &args, &all_regions, &continents, &countries, &highway_systems, &traveler_lists, &clin_db_val, &updates, &systemupdates); + #endif + sqlfile2(&et, &args, &graph_types, &graph_vector, datacheckerrors); + } // print some statistics cout << et.et() << "Processed " << highway_systems.size() << " highway systems." << endl; From 96be2a38300bf8ea63c23ddd17cf80441669fd9f Mon Sep 17 00:00:00 2001 From: eric bryant Date: Thu, 27 Jun 2019 22:26:40 -0400 Subject: [PATCH 2/3] 1 function for all 3 master graphs C++ flavor runs concurrent with subgraphs --- .../classes/GraphGeneration/HighwayGraph.cpp | 92 +++++++++++++++++++ .../cplusplus/functions/graph_generation.cpp | 39 +++----- siteupdate/cplusplus/functions/sql_file.cpp | 2 + siteupdate/cplusplus/siteupdate.cpp | 2 +- .../cplusplus/threads/MasterTmgThread.cpp | 6 ++ .../cplusplus/threads/MasterTmgThreads.cpp | 7 -- siteupdate/python-teresco/siteupdate.py | 87 +++++++++++++++--- 7 files changed, 187 insertions(+), 48 deletions(-) create mode 100644 siteupdate/cplusplus/threads/MasterTmgThread.cpp delete mode 100644 siteupdate/cplusplus/threads/MasterTmgThreads.cpp diff --git a/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp b/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp index 8d15c2a2..f0085cd2 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp +++ b/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp @@ -357,6 +357,98 @@ class HighwayGraph traveler_lists.sort(sort_travelers_by_name); } + // write the entire set of highway data in .tmg format. + // The first line is a header specifying the format and version number, + // The second line specifies the number of waypoints, w, the number of connections, c, + // and for traveled graphs only, the number of travelers. + // Then, w lines describing waypoints (label, latitude, longitude). + // Then, c lines describing connections (endpoint 1 number, endpoint 2 number, route label), + // followed on traveled graphs only by a hexadecimal code encoding travelers on that segment, + // followed on both collapsed & traveled graphs by a list of latitude & longitude values + // for intermediate "shaping points" along the edge, ordered from endpoint 1 to endpoint 2. + // + void write_master_graphs_tmg(std::vector &graph_vector, std::string path, std::list &traveler_lists) + { std::ofstream simplefile(path+"tm-master-simple.tmg"); + std::ofstream collapfile(path+"tm-master-collapsed.tmg"); + std::ofstream travelfile(path+"tm-master-traveled.tmg"); + unsigned int num_collapsed_edges = collapsed_edge_count(); + unsigned int num_traveled_edges = traveled_edge_count(); + simplefile << "TMG 1.0 simple\n"; + collapfile << "TMG 1.0 collapsed\n"; + travelfile << "TMG 2.0 traveled\n"; + simplefile << vertices.size() << ' ' << simple_edge_count() << '\n'; + collapfile << num_collapsed_vertices() << ' ' << num_collapsed_edges << '\n'; + travelfile << num_traveled_vertices() << ' ' << num_traveled_edges << ' ' << traveler_lists.size() << '\n'; + + // write vertices + unsigned int sv = 0; + unsigned int cv = 0; + unsigned int tv = 0; + for (std::pair wv : vertices) + { char fstr[57]; + sprintf(fstr, " %.15g %.15g", wv.second->lat, wv.second->lng); + // all vertices for simple graph + simplefile << *(wv.second->unique_name) << fstr << '\n'; + wv.second->s_vertex_num[0] = sv; + sv++; + // visible vertices... + if (wv.second->visibility >= 1) + { // for traveled graph, + travelfile << *(wv.second->unique_name) << fstr << '\n'; + wv.second->t_vertex_num[0] = tv; + tv++; + if (wv.second->visibility == 2) + { // and for collapsed graph + collapfile << *(wv.second->unique_name) << fstr << '\n'; + wv.second->c_vertex_num[0] = cv; + cv++; + } + } + } + // now edges, only write if not already written + for (std::pair wv : vertices) + { for (HGEdge *e : wv.second->incident_s_edges) + if (!e->s_written) + { e->s_written = 1; + simplefile << e->vertex1->s_vertex_num[0] << ' ' << e->vertex2->s_vertex_num[0] << ' ' << e->label(0) << '\n'; + } + // write edges if vertex is visible... + if (wv.second->visibility >= 1) + { // in traveled graph, + for (HGEdge *e : wv.second->incident_t_edges) + if (!e->t_written) + { e->t_written = 1; + travelfile << e->traveled_tmg_line(0, &traveler_lists, 0) << '\n'; + } + if (wv.second->visibility == 2) + { // and in collapsed graph + for (HGEdge *e : wv.second->incident_c_edges) + if (!e->c_written) + { e->c_written = 1; + collapfile << e->collapsed_tmg_line(0, 0) << '\n'; + } + } + } + } + // traveler names + for (TravelerList *t : traveler_lists) + travelfile << t->traveler_name << ' '; + travelfile << '\n'; + simplefile.close(); + collapfile.close(); + travelfile.close(); + + graph_vector[0].vertices = vertices.size(); + graph_vector[1].vertices = num_collapsed_vertices(); + graph_vector[2].vertices = num_traveled_vertices(); + graph_vector[0].edges = simple_edge_count(); + graph_vector[1].edges = num_collapsed_edges; + graph_vector[2].edges = num_traveled_edges; + graph_vector[0].travelers = 0; + graph_vector[1].travelers = 0; + graph_vector[2].travelers = traveler_lists.size(); + } + // write the entire set of highway data in .tmg format. // The first line is a header specifying // the format and version number, the second line specifying the diff --git a/siteupdate/cplusplus/functions/graph_generation.cpp b/siteupdate/cplusplus/functions/graph_generation.cpp index 825728f7..f821c2da 100644 --- a/siteupdate/cplusplus/functions/graph_generation.cpp +++ b/siteupdate/cplusplus/functions/graph_generation.cpp @@ -28,33 +28,13 @@ else { list *regions; 'c', 'M', (list*)0, (list*)0, (PlaceRadius*)0); graph_vector.emplace_back("tm-master", "All Travel Mapping Data", 't', 'M', (list*)0, (list*)0, (PlaceRadius*)0); - - /*#ifdef threading_enabled - if (args.numthreads <= 1) - #endif//*/ - { cout << et.et() << "Writing master TM simple graph file, tm-master-simple.tmg" << endl; - graph_data.write_master_tmg_simple(&graph_vector[0], args.graphfilepath+"/tm-master-simple.tmg"); - cout << et.et() << "Writing master TM collapsed graph file, tm-master-collapsed.tmg." << endl; - graph_data.write_master_tmg_collapsed(&graph_vector[1], args.graphfilepath+"/tm-master-collapsed.tmg", 0); - cout << et.et() << "Writing master TM traveled graph file, tm-master-traveled.tmg." << endl; - graph_data.write_master_tmg_traveled(&graph_vector[2], args.graphfilepath+"/tm-master-traveled.tmg", &traveler_lists, 0); - } - /*#ifdef threading_enabled - else { cout << et.et() << "Writing master TM simple graph file, tm-master-simple.tmg" << endl; - thr[0] = new thread(MasterTmgSimpleThread, &graph_data, &graph_vector[0], args.graphfilepath+"/tm-master-simple.tmg"); - cout << et.et() << "Writing master TM collapsed graph file, tm-master-collapsed.tmg." << endl; - thr[1] = new thread(MasterTmgCollapsedThread, &graph_data, &graph_vector[1], args.graphfilepath+"/tm-master-collapsed.tmg"); - thr[0]->join(); - thr[1]->join(); - delete thr[0]; - delete thr[1]; - } - #endif//*/ - graph_types.push_back({"master", "All Travel Mapping Data", "These graphs contain all routes currently plotted in the Travel Mapping project."}); size_t graphnum = 3; - #ifdef threading_enabled - cout << et.et() << "Setting up subgraphs." << flush; + #ifndef threading_enabled + cout << et.et() << "Writing master TM graph files." << endl; + graph_data.write_master_graphs_tmg(graph_vector, args.graphfilepath + "/", traveler_lists); + #else + cout << et.et() << "Setting up subgraphs." << endl; #endif #include "subgraphs/continent.cpp" #include "subgraphs/multisystem.cpp" @@ -64,10 +44,13 @@ else { list *regions; #include "subgraphs/area.cpp" #include "subgraphs/region.cpp" #ifdef threading_enabled - // write new graph_vector entries to disk + // write graph_vector entries to disk + cout << et.et() << "Writing master TM graph files." << flush; + thr[0] = new thread(MasterTmgThread, &graph_data, &graph_vector, args.graphfilepath+'/', &traveler_lists, &graphnum, &list_mtx, &all_waypoints, &et); // set up for threaded subgraph generation - for (unsigned int t = 0; t < args.numthreads; t++) - thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath + "/", &all_waypoints, &et); + // start at t=1, because MasterTmgThread will spawn another SubgraphThread when finished + for (unsigned int t = 1; t < args.numthreads; t++) + thr[t] = new thread(SubgraphThread, t, &graph_data, &graph_vector, &graphnum, &list_mtx, args.graphfilepath+'/', &all_waypoints, &et); for (unsigned int t = 0; t < args.numthreads; t++) thr[t]->join(); for (unsigned int t = 0; t < args.numthreads; t++) diff --git a/siteupdate/cplusplus/functions/sql_file.cpp b/siteupdate/cplusplus/functions/sql_file.cpp index 0542639e..d3db49d2 100644 --- a/siteupdate/cplusplus/functions/sql_file.cpp +++ b/siteupdate/cplusplus/functions/sql_file.cpp @@ -351,7 +351,9 @@ void sqlfile1 } sqlfile << ";\n"; sqlfile.close(); + #ifdef threading_enabled std::cout << '\n' + et->et() + "Pause writing database file " + args->databasename + ".sql.\n" << std::flush; + #endif } void sqlfile2 diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index a54b306b..e25dbea8 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -82,7 +82,7 @@ class HGEdge; #include "threads/ComputeStatsThread.cpp" #include "threads/UserLogThread.cpp" #include "threads/SubgraphThread.cpp" -#include "threads/MasterTmgThreads.cpp" +#include "threads/MasterTmgThread.cpp" #include "functions/sql_file.cpp" using namespace std; diff --git a/siteupdate/cplusplus/threads/MasterTmgThread.cpp b/siteupdate/cplusplus/threads/MasterTmgThread.cpp new file mode 100644 index 00000000..13e91c79 --- /dev/null +++ b/siteupdate/cplusplus/threads/MasterTmgThread.cpp @@ -0,0 +1,6 @@ +void MasterTmgThread( HighwayGraph *graph_data, std::vector *graph_vector, + std::string path, std::list *traveler_lists, + size_t *index, std::mutex *mtx, WaypointQuadtree *qt, ElapsedTime *et) +{ graph_data->write_master_graphs_tmg(*graph_vector, path, *traveler_lists); + SubgraphThread(0, graph_data, graph_vector, index, mtx, path, qt, et); +} diff --git a/siteupdate/cplusplus/threads/MasterTmgThreads.cpp b/siteupdate/cplusplus/threads/MasterTmgThreads.cpp deleted file mode 100644 index ad8e0c14..00000000 --- a/siteupdate/cplusplus/threads/MasterTmgThreads.cpp +++ /dev/null @@ -1,7 +0,0 @@ -void MasterTmgSimpleThread(HighwayGraph *graph_data, GraphListEntry *msptr, std::string filename) -{ graph_data->write_master_tmg_simple(msptr, filename); -} - -void MasterTmgCollapsedThread(HighwayGraph *graph_data, GraphListEntry *mcptr, std::string filename) -{ graph_data->write_master_tmg_collapsed(mcptr, filename, 1); -} diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index d0be3d66..aa7ad571 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1879,6 +1879,79 @@ def matching_traveled_edges(self, mv, regions=None, systems=None, trav_set.add(t) return (edge_set, sorted(trav_set, key=lambda TravelerList: TravelerList.traveler_name)) + # write the entire set of highway data in .tmg format. + # The first line is a header specifying the format and version number, + # The second line specifies the number of waypoints, w, the number of connections, c, + # and for traveled graphs only, the number of travelers. + # Then, w lines describing waypoints (label, latitude, longitude). + # Then, c lines describing connections (endpoint 1 number, endpoint 2 number, route label), + # followed on traveled graphs only by a hexadecimal code encoding travelers on that segment, + # followed on both collapsed & traveled graphs by a list of latitude & longitude values + # for intermediate "shaping points" along the edge, ordered from endpoint 1 to endpoint 2. + # + def write_master_graphs_tmg(self, graph_list, path, traveler_lists): + simplefile = open(path+"tm-master-simple.tmg","w",encoding='utf-8') + collapfile = open(path+"tm-master-collapsed.tmg","w",encoding='utf-8') + travelfile = open(path+"tm-master-traveled.tmg","w",encoding='utf-8') + num_collapsed_edges = self.collapsed_edge_count() + num_traveled_edges = self.traveled_edge_count() + simplefile.write("TMG 1.0 simple\n") + collapfile.write("TMG 1.0 collapsed\n") + travelfile.write("TMG 2.0 traveled\n") + simplefile.write(str(len(self.vertices)) + ' ' + str(self.simple_edge_count()) + '\n') + collapfile.write(str(self.num_collapsed_vertices()) + ' ' + str(num_collapsed_edges) + '\n') + travelfile.write(str(self.num_traveled_vertices()) + ' ' + str(num_traveled_edges) + ' ' + str(len(traveler_lists)) + '\n') + + # write vertices + sv = 0 + cv = 0 + tv = 0 + for v in self.vertices.values(): + # all vertices for simple graph + simplefile.write(v.unique_name+' '+str(v.lat)+' '+str(v.lng)+'\n') + v.s_vertex_num = sv + sv += 1 + # visible vertices... + if v.visibility >= 1: + # for traveled graph, + travelfile.write(v.unique_name+' '+str(v.lat)+' '+str(v.lng)+'\n') + v.t_vertex_num = tv + tv += 1 + if v.visibility == 2: + # and for collapsed graph + collapfile.write(v.unique_name+' '+str(v.lat)+' '+str(v.lng)+'\n') + v.c_vertex_num = cv + cv += 1 + # now edges, only write if not already written + for v in self.vertices.values(): + for e in v.incident_s_edges: + if not e.s_written: + e.s_written = 1 + simplefile.write(str(e.vertex1.s_vertex_num) + ' ' + str(e.vertex2.s_vertex_num) + ' ' + e.label() + '\n') + # write edges if vertex is visible... + if v.visibility >= 1: + # in traveled graph, + for e in v.incident_t_edges: + if not e.t_written: + e.t_written = 1 + travelfile.write(e.traveled_tmg_line(traveler_lists) + '\n') + if v.visibility == 2: + # and in collapsed graph + for e in v.incident_c_edges: + if not e.c_written: + e.c_written = 1 + collapfile.write(e.collapsed_tmg_line() + '\n') + # traveler names + for t in traveler_lists: + travelfile.write(t.traveler_name + ' ') + travelfile.write('\n') + simplefile.close() + collapfile.close() + travelfile.close() + graph_list.append(GraphListEntry('tm-master-simple.tmg', 'All Travel Mapping Data', sv, self.simple_edge_count(), 0, 'simple', 'master')) + graph_list.append(GraphListEntry('tm-master-collapsed.tmg', 'All Travel Mapping Data', cv, self.collapsed_edge_count(), 0, 'collapsed', 'master')) + graph_list.append(GraphListEntry('tm-master-traveled.tmg', 'All Travel Mapping Data', tv, self.traveled_edge_count(), len(traveler_lists), 'traveled', 'master')) + # write the entire set of highway data a format very similar to # the original .gra format. The first line is a header specifying # the format and version number, the second line specifying the @@ -3088,18 +3161,8 @@ def run(self): if args.skipgraphs or args.errorcheck: print(et.et() + "SKIPPING generation of subgraphs.", flush=True) else: - print(et.et() + "Writing master TM simple graph file, tm-master-simple.tmg", flush=True) - (sv, se) = graph_data.write_master_tmg_simple(args.graphfilepath+'/tm-master-simple.tmg') - graph_list.append(GraphListEntry('tm-master-simple.tmg', 'All Travel Mapping Data', sv, se, 0, 'simple', 'master')) - - print(et.et() + "Writing master TM collapsed graph file, tm-master-collapsed.tmg.", flush=True) - (cv, ce) = graph_data.write_master_tmg_collapsed(args.graphfilepath+'/tm-master-collapsed.tmg') - graph_list.append(GraphListEntry('tm-master-collapsed.tmg', 'All Travel Mapping Data', cv, ce, 0, 'collapsed', 'master')) - - print(et.et() + "Writing master TM traveled graph file, tm-master-traveled.tmg.", flush=True) - (tv, te, tt) = graph_data.write_master_tmg_traveled(args.graphfilepath+'/tm-master-traveled.tmg', traveler_lists) - graph_list.append(GraphListEntry('tm-master-traveled.tmg', 'All Travel Mapping Data', tv, te, tt, 'traveled', 'master')) - + print(et.et() + "Writing master TM graph files.", flush=True) + graph_data.write_master_graphs_tmg(graph_list, args.graphfilepath+'/', traveler_lists) graph_types.append(['master', 'All Travel Mapping Data', 'These graphs contain all routes currently plotted in the Travel Mapping project.']) From 336fb3d505cb3ab98110c6d21a930effbf84a394 Mon Sep 17 00:00:00 2001 From: eric bryant Date: Thu, 27 Jun 2019 22:48:01 -0400 Subject: [PATCH 3/3] remove old unused master graph functions --- .../classes/GraphGeneration/HighwayGraph.cpp | 129 ------------------ siteupdate/python-teresco/siteupdate.py | 118 ---------------- 2 files changed, 247 deletions(-) diff --git a/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp b/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp index f0085cd2..df80067b 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp +++ b/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp @@ -449,135 +449,6 @@ class HighwayGraph graph_vector[2].travelers = traveler_lists.size(); } - // write the entire set of highway data in .tmg format. - // The first line is a header specifying - // the format and version number, the second line specifying the - // number of waypoints, w, and the number of connections, c, then w - // lines describing waypoints (label, latitude, longitude), then c - // lines describing connections (endpoint 1 number, endpoint 2 - // number, route label) - // - // passes number of vertices and number of edges written by reference - // - void write_master_tmg_simple(GraphListEntry *msptr, std::string filename) - { std::ofstream tmgfile(filename.data()); - tmgfile << "TMG 1.0 simple\n"; - tmgfile << vertices.size() << ' ' << simple_edge_count() << '\n'; - // number waypoint entries as we go - int s_vertex_num = 0; - for (std::pair wv : vertices) - { char fstr[56]; - sprintf(fstr, "%.15g %.15g", wv.second->lat, wv.second->lng); - tmgfile << *(wv.second->unique_name) << ' ' << fstr << '\n'; - wv.second->s_vertex_num[0] = s_vertex_num; - s_vertex_num++; - } - // sanity check - if (vertices.size() != s_vertex_num) - std::cout << "ERROR: computed " << vertices.size() << " waypoints but wrote " << s_vertex_num << std::endl; - - // now edges, only print if not already printed - int edge = 0; - for (std::pair wv : vertices) - for (HGEdge *e : wv.second->incident_s_edges) - if (!e->s_written) - { e->s_written = 1; - tmgfile << e->vertex1->s_vertex_num[0] << ' ' << e->vertex2->s_vertex_num[0] << ' ' << e->label(0) << '\n'; - edge++; - } - // sanity checks - for (std::pair wv : vertices) - for (HGEdge *e : wv.second->incident_s_edges) - if (!e->s_written) - std::cout << "ERROR: never wrote edge " << e->vertex1->s_vertex_num << ' ' << e->vertex2->s_vertex_num[0] << ' ' << e->label(0) << std::endl; - if (simple_edge_count() != edge) - std::cout << "ERROR: computed " << simple_edge_count() << " edges but wrote " << edge << std::endl; - - tmgfile.close(); - msptr->vertices = vertices.size(); - msptr->edges = simple_edge_count(); - msptr->travelers = 0; - } - - // write the entire set of data in the tmg collapsed edge format - void write_master_tmg_collapsed(GraphListEntry *mcptr, std::string filename, unsigned int threadnum) - { std::ofstream tmgfile(filename.data()); - unsigned int num_collapsed_edges = collapsed_edge_count(); - tmgfile << "TMG 1.0 collapsed\n"; - tmgfile << num_collapsed_vertices() << " " << num_collapsed_edges << '\n'; - - // write visible vertices - int c_vertex_num = 0; - for (std::pair wv : vertices) - if (wv.second->visibility == 2) - { char fstr[56]; - sprintf(fstr, "%.15g %.15g", wv.second->lat, wv.second->lng); - tmgfile << *(wv.second->unique_name) << ' ' << fstr << '\n'; - wv.second->c_vertex_num[threadnum] = c_vertex_num; - c_vertex_num++; - } - // write collapsed edges - int edge = 0; - for (std::pair wv : vertices) - if (wv.second->visibility == 2) - for (HGEdge *e : wv.second->incident_c_edges) - if (!e->c_written) - { e->c_written = 1; - tmgfile << e->collapsed_tmg_line(0, threadnum) << '\n'; - edge++; - } - // sanity check on edges written - if (num_collapsed_edges != edge) - std::cout << "ERROR: computed " << num_collapsed_edges << " collapsed edges, but wrote " << edge << '\n'; - - tmgfile.close(); - mcptr->vertices = num_collapsed_vertices(); - mcptr->edges = num_collapsed_edges; - mcptr->travelers = 0; - } - - // write the entire set of data in the tmg traveled format - void write_master_tmg_traveled(GraphListEntry *mtptr, std::string filename, std::list *traveler_lists, unsigned int threadnum) - { std::ofstream tmgfile(filename.data()); - unsigned int num_traveled_edges = traveled_edge_count(); - tmgfile << "TMG 2.0 traveled\n"; - tmgfile << num_traveled_vertices() << ' ' << num_traveled_edges << ' ' << traveler_lists->size() << '\n'; - - // write visible vertices - int t_vertex_num = 0; - for (std::pair wv : vertices) - if (wv.second->visibility >= 1) - { char fstr[56]; - sprintf(fstr, "%.15g %.15g", wv.second->lat, wv.second->lng); - tmgfile << *(wv.second->unique_name) << ' ' << fstr << '\n'; - wv.second->t_vertex_num[threadnum] = t_vertex_num; - t_vertex_num++; - } - // write traveled edges - int edge = 0; - for (std::pair wv : vertices) - if (wv.second->visibility >= 1) - for (HGEdge *e : wv.second->incident_t_edges) - if (!e->t_written) - { e->t_written = 1; - tmgfile << e->traveled_tmg_line(0, traveler_lists, threadnum) << '\n'; - edge++; - } - // traveler names - for (TravelerList *t : *traveler_lists) - tmgfile << t->traveler_name << ' '; - tmgfile << '\n'; - - // sanity check on edges written - if (num_traveled_edges != edge) - std::cout << "ERROR: computed " << num_traveled_edges << " traveled edges, but wrote " << edge << '\n'; - - tmgfile.close(); - mtptr->vertices = num_traveled_vertices(); - mtptr->edges = num_traveled_edges; - mtptr->travelers = traveler_lists->size(); - } - // write a subset of the data, // in simple, collapsed and traveled formats, // restricted by regions in the list if given, diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index aa7ad571..dcc7b4d9 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1952,124 +1952,6 @@ def write_master_graphs_tmg(self, graph_list, path, traveler_lists): graph_list.append(GraphListEntry('tm-master-collapsed.tmg', 'All Travel Mapping Data', cv, self.collapsed_edge_count(), 0, 'collapsed', 'master')) graph_list.append(GraphListEntry('tm-master-traveled.tmg', 'All Travel Mapping Data', tv, self.traveled_edge_count(), len(traveler_lists), 'traveled', 'master')) - # write the entire set of highway data a format very similar to - # the original .gra format. The first line is a header specifying - # the format and version number, the second line specifying the - # number of waypoints, w, and the number of connections, c, then w - # lines describing waypoints (label, latitude, longitude), then c - # lines describing connections (endpoint 1 number, endpoint 2 - # number, route label) - # - # returns tuple of number of vertices and number of edges written - # - def write_master_tmg_simple(self,filename): - tmgfile = open(filename, 'w') - tmgfile.write("TMG 1.0 simple\n") - tmgfile.write(str(len(self.vertices)) + ' ' + str(self.simple_edge_count()) + '\n') - # number waypoint entries as we go to support original .gra - # format output - s_vertex_num = 0 - for v in self.vertices.values(): - tmgfile.write(v.unique_name + ' ' + str(v.lat) + ' ' + str(v.lng) + '\n') - v.s_vertex_num = s_vertex_num - s_vertex_num += 1 - - # sanity check - if len(self.vertices) != s_vertex_num: - print("ERROR: computed " + str(len(self.vertices)) + " waypoints but wrote " + str(s_vertex_num)) - - # now edges, only print if not already printed - edge = 0 - for v in self.vertices.values(): - for e in v.incident_s_edges: - if not e.s_written: - e.s_written = True - tmgfile.write(str(e.vertex1.s_vertex_num) + ' ' + str(e.vertex2.s_vertex_num) + ' ' + e.label() + '\n') - edge += 1 - - # sanity checks - for v in self.vertices.values(): - for e in v.incident_s_edges: - if not e.s_written: - print("ERROR: never wrote edge " + str(e.vertex1.s_vertex_num) + ' ' + str(e.vertex2.s_vertex_num) + ' ' + e.label() + '\n') - if self.simple_edge_count() != edge: - print("ERROR: computed " + str(self.simple_edge_count()) + " edges but wrote " + str(edge) + "\n") - - tmgfile.close() - return (len(self.vertices), self.simple_edge_count()) - - # write the entire set of data in the tmg collapsed edge format - def write_master_tmg_collapsed(self, filename): - tmgfile = open(filename, 'w') - tmgfile.write("TMG 1.0 collapsed\n") - tmgfile.write(str(self.num_collapsed_vertices()) + " " + - str(self.collapsed_edge_count()) + "\n") - - # write visible vertices - c_vertex_num = 0 - for v in self.vertices.values(): - if v.visibility == 2: - v.c_vertex_num = c_vertex_num - tmgfile.write(v.unique_name + ' ' + str(v.lat) + ' ' + str(v.lng) + '\n') - c_vertex_num += 1 - - # write collapsed edges - edge = 0 - for v in self.vertices.values(): - if v.visibility == 2: - for e in v.incident_c_edges: - if not e.c_written: - e.c_written = True - tmgfile.write(e.collapsed_tmg_line() + '\n') - edge += 1 - - # sanity check on edges written - if self.collapsed_edge_count() != edge: - print("ERROR: computed " + str(self.collapsed_edge_count()) + " collapsed edges, but wrote " + str(edge) + "\n") - - tmgfile.close() - return (self.num_collapsed_vertices(), self.collapsed_edge_count()) - - # write the entire set of data in the tmg traveled format - def write_master_tmg_traveled(self, filename, traveler_lists): - tmgfile = open(filename, 'w') - tmgfile.write("TMG 2.0 traveled\n") - tmgfile.write(str(self.num_traveled_vertices()) + " " + - str(self.traveled_edge_count()) + " " + - str(len(traveler_lists)) + "\n") - - # write visible vertices - t_vertex_num = 0 - for v in self.vertices.values(): - if v.visibility >= 1: - v.t_vertex_num = t_vertex_num - tmgfile.write(v.unique_name + ' ' + str(v.lat) + ' ' + str(v.lng) + '\n') - t_vertex_num += 1 - - # write traveled edges - edge = 0 - for v in self.vertices.values(): - if v.visibility >= 1: - for e in v.incident_t_edges: - if not e.t_written: - e.t_written = True - tmgfile.write(e.traveled_tmg_line(traveler_lists) + '\n') - edge += 1 - - # traveler names - for t in traveler_lists: - tmgfile.write(t.traveler_name + ' ') - tmgfile.write('\n') - - # sanity check on edges written - if self.traveled_edge_count() != edge: - print("ERROR: computed " + str(self.traveled_edge_count()) + " traveled edges, but wrote " + str(edge) + "\n") - - tmgfile.close() - return (self.num_traveled_vertices(), - self.traveled_edge_count(), - len(traveler_lists)) - # write a subset of the data, # in simple, collapsed and traveled formats, # restricted by regions in the list if given,