From 079c744c8d3e16da1d16d60e3064ed24c311b076 Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 19 Jul 2021 15:40:36 -0400 Subject: [PATCH 01/10] Simplify collapsed graph file names for #450 --- .../classes/GraphGeneration/GraphListEntry.cpp | 2 +- .../classes/GraphGeneration/HighwayGraph.cpp | 2 +- siteupdate/python-teresco/siteupdate.py | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/siteupdate/cplusplus/classes/GraphGeneration/GraphListEntry.cpp b/siteupdate/cplusplus/classes/GraphGeneration/GraphListEntry.cpp index caeac56f..e6ec560d 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/GraphListEntry.cpp +++ b/siteupdate/cplusplus/classes/GraphGeneration/GraphListEntry.cpp @@ -20,7 +20,7 @@ GraphListEntry::GraphListEntry(std::string r, std::string d, char f, char c, std std::string GraphListEntry::filename() { switch (form) { case 's': return root+"-simple.tmg"; - case 'c': return root+"-collapsed.tmg"; + case 'c': return root+".tmg"; case 't': return root+"-traveled.tmg"; default : return std::string("ERROR: GraphListEntry::filename() unexpected format token ('")+form+"')"; } diff --git a/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp b/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp index c16f8221..9bb4038d 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp +++ b/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp @@ -300,7 +300,7 @@ inline void HighwayGraph::matching_vertices_and_edges // void HighwayGraph::write_master_graphs_tmg() { std::ofstream simplefile(Args::graphfilepath + "/tm-master-simple.tmg"); - std::ofstream collapfile(Args::graphfilepath + "/tm-master-collapsed.tmg"); + std::ofstream collapfile(Args::graphfilepath + "/tm-master.tmg"); std::ofstream travelfile(Args::graphfilepath + "/tm-master-traveled.tmg"); simplefile << "TMG 1.0 simple\n"; collapfile << "TMG 1.0 collapsed\n"; diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 710cc4a8..2ca81cea 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2310,7 +2310,7 @@ def matching_vertices_and_edges(self, qt, regions, systems, placeradius, rg_vset # def write_master_graphs_tmg(self, graph_list, path, traveler_lists): simplefile = open(path+"tm-master-simple.tmg","w",encoding='utf-8') - collapfile = open(path+"tm-master-collapsed.tmg","w",encoding='utf-8') + collapfile = open(path+"tm-master.tmg","w",encoding='utf-8') travelfile = open(path+"tm-master-traveled.tmg","w",encoding='utf-8') cv = 0 tv = 0 @@ -2387,7 +2387,7 @@ def write_master_graphs_tmg(self, graph_list, path, traveler_lists): collapfile.close() travelfile.close() graph_list.append(GraphListEntry('tm-master-simple.tmg', 'All Travel Mapping Data', sv, se, 0, 'simple', 'master')) - graph_list.append(GraphListEntry('tm-master-collapsed.tmg', 'All Travel Mapping Data', cv, ce, 0, 'collapsed', 'master')) + graph_list.append(GraphListEntry('tm-master.tmg', 'All Travel Mapping Data', cv, ce, 0, 'collapsed', 'master')) graph_list.append(GraphListEntry('tm-master-traveled.tmg', 'All Travel Mapping Data', tv, te, len(traveler_lists), 'traveled', 'master')) # print summary info print(" Simple graph has " + str(len(self.vertices)) + @@ -2404,7 +2404,7 @@ def write_master_graphs_tmg(self, graph_list, path, traveler_lists): # or to within a given area if placeradius is given def write_subgraphs_tmg(self, graph_list, path, root, descr, category, regions, systems, placeradius, qt): simplefile = open(path+root+"-simple.tmg","w",encoding='utf-8') - collapfile = open(path+root+"-collapsed.tmg","w",encoding='utf-8') + collapfile = open(path+root+".tmg","w",encoding='utf-8') travelfile = open(path+root+"-traveled.tmg","w",encoding='utf-8') (mv, cv_count, tv_count, mse, mce, mte, traveler_lists) = self.matching_vertices_and_edges(qt, regions, systems, placeradius, self.rg_vset_hash) """if len(traveler_lists) == 0: @@ -2462,9 +2462,9 @@ def write_subgraphs_tmg(self, graph_list, path, root, descr, category, regions, collapfile.close() travelfile.close() - graph_list.append(GraphListEntry(root+ "-simple.tmg", descr, len(mv), len(mse), 0, "simple", category)) - graph_list.append(GraphListEntry(root+"-collapsed.tmg", descr, cv_count, len(mce), 0, "collapsed", category)) - graph_list.append(GraphListEntry(root+ "-traveled.tmg", descr, tv_count, len(mte), len(traveler_lists), "traveled", category)) + graph_list.append(GraphListEntry(root+ "-simple.tmg", descr, len(mv), len(mse), 0, "simple", category)) + graph_list.append(GraphListEntry(root+ ".tmg", descr, cv_count, len(mce), 0, "collapsed", category)) + graph_list.append(GraphListEntry(root+"-traveled.tmg", descr, tv_count, len(mte), len(traveler_lists), "traveled", category)) def format_clinched_mi(clinched,total): """return a nicely-formatted string for a given number of miles From 39411bb01f897f5dd7e978def580fdeb1c40618f Mon Sep 17 00:00:00 2001 From: eric bryant Date: Mon, 19 Jul 2021 14:05:32 -0400 Subject: [PATCH 02/10] matching_vertices_and_edges comment for yakra#177 --- siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp | 2 +- siteupdate/python-teresco/siteupdate.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp b/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp index 9bb4038d..2f8cbbd1 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp +++ b/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp @@ -203,7 +203,7 @@ inline void HighwayGraph::matching_vertices_and_edges if (g.placeradius) pvset = g.placeradius->vertices(qt, this); - // determine which vertices are within our region(s) and/or system(s) + // determine which vertices are within our PlaceRadius, region(s) and/or system(s) if (g.regions) { mvset = rvset; if (g.placeradius) mvset = mvset & pvset; diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 2ca81cea..61d55fae 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2233,7 +2233,7 @@ def matching_vertices_and_edges(self, qt, regions, systems, placeradius, rg_vset if placeradius is not None: pvset = placeradius.vertices(qt, self) - # determine which vertices are within our region(s) and/or system(s) + # determine which vertices are within our PlaceRadius, region(s) and/or system(s) if regions is not None: mvset = rvset if placeradius is not None: From bf525b59a6a4b382d435d8ef0c6c796eb672fcad Mon Sep 17 00:00:00 2001 From: eric bryant Date: Mon, 19 Jul 2021 14:49:33 -0400 Subject: [PATCH 03/10] concurrency detection rework for #449 * speedup; find all concurrencies on 1st pass; ignore segments on subsequent passes * prevent memory leak from overwritten/orphaned concurrency lists * "reverse" check in HGEdge ctor no longer needed interactive rebase d54bfbd89ced77e04655b9ac5a971b01551c59a5 --- .../classes/GraphGeneration/HGEdge.cpp | 12 ----- siteupdate/cplusplus/siteupdate.cpp | 1 - .../cplusplus/tasks/concurrency_detection.cpp | 51 +++++++++---------- siteupdate/python-teresco/siteupdate.py | 47 +++++++---------- 4 files changed, 40 insertions(+), 71 deletions(-) diff --git a/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.cpp b/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.cpp index 0cf9624d..f4fe90ee 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.cpp +++ b/siteupdate/cplusplus/classes/GraphGeneration/HGEdge.cpp @@ -20,18 +20,6 @@ HGEdge::HGEdge(HighwaySegment *s, HighwayGraph *graph, int numthreads) t_written[0] = 0; vertex1 = s->waypoint1->hashpoint()->vertex; vertex2 = s->waypoint2->hashpoint()->vertex; - // checks for the very unusual cases where an edge ends up - // in the system as itself and its "reverse" - for (HGEdge *e : vertex1->incident_s_edges) - if (e->vertex1 == vertex2 && e->vertex2 == vertex1) - { delete this; - return; - } - for (HGEdge *e : vertex2->incident_s_edges) - if (e->vertex1 == vertex2 && e->vertex2 == vertex1) - { delete this; - return; - } format = simple | collapsed | traveled; segment_name = s->segment_name(); vertex1->incident_s_edges.push_back(this); diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index 2cb0c2df..49bc9008 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -467,7 +467,6 @@ int main(int argc, char *argv[]) // now augment any traveler clinched segments for concurrencies cout << et.et() << "Augmenting travelers for detected concurrent segments." << flush; - //#include "debug/concurrency_augments.cpp" #ifdef threading_enabled list* augment_lists = new list[Args::numthreads]; TravelerList::tl_it = TravelerList::allusers.begin(); diff --git a/siteupdate/cplusplus/tasks/concurrency_detection.cpp b/siteupdate/cplusplus/tasks/concurrency_detection.cpp index f347e13d..451ab5e4 100644 --- a/siteupdate/cplusplus/tasks/concurrency_detection.cpp +++ b/siteupdate/cplusplus/tasks/concurrency_detection.cpp @@ -8,34 +8,29 @@ for (HighwaySystem *h : HighwaySystem::syslist) { cout << '.' << flush; for (Route *r : h->route_list) for (HighwaySegment *s : r->segment_list) - if (s->waypoint1->colocated && s->waypoint2->colocated) - for ( Waypoint *w1 : *(s->waypoint1->colocated) ) - if (w1->route != r) - for ( Waypoint *w2 : *(s->waypoint2->colocated) ) - if (w1->route == w2->route) - { HighwaySegment *other = w1->route->find_segment_by_waypoints(w1,w2); - if (other) - if (!s->concurrent) - { s->concurrent = new list; - // deleted on termination of program - other->concurrent = s->concurrent; - s->concurrent->push_back(s); - s->concurrent->push_back(other); - concurrencyfile << "New concurrency [" << s->str() << "][" << other->str() << "] (" << s->concurrent->size() << ")\n"; - } - else if (!contains(*s->concurrent, other)) - { other->concurrent = s->concurrent; - s->concurrent->push_back(other); - //concurrencyfile << "Added concurrency [" << s->str() << "]-[" \ - << other->str() << "] (" << s->concurrent->size() << ")\n"; - concurrencyfile << "Extended concurrency "; - for (HighwaySegment *x : *(s->concurrent)) - concurrencyfile << '[' << x->str() << ']'; - concurrencyfile << " (" << s->concurrent->size() << ")\n"; - } - } - // see https://github.com/TravelMapping/DataProcessing/issues/137 - // changes not yet implemented in either the original Python or this C++ version. + if (!s->concurrent && s->waypoint1->colocated && s->waypoint2->colocated) + for ( Waypoint *w1 : *(s->waypoint1->colocated) ) + for ( Waypoint *w2 : *(s->waypoint2->colocated) ) + if (w1->route == w2->route && (w1 != s->waypoint1 || w2 != s->waypoint2) && (w1 != s->waypoint2 || w2 != s->waypoint1)) + { HighwaySegment *other = w1->route->find_segment_by_waypoints(w1,w2); + if (other) + if (!s->concurrent) + { s->concurrent = new list; + // deleted by ~HighwaySegment + other->concurrent = s->concurrent; + s->concurrent->push_back(s); + s->concurrent->push_back(other); + concurrencyfile << "New concurrency [" << s->str() << "][" << other->str() << "] (2)\n"; + } + else + { other->concurrent = s->concurrent; + s->concurrent->push_back(other); + concurrencyfile << "Extended concurrency "; + for (HighwaySegment *x : *(s->concurrent)) + concurrencyfile << '[' << x->str() << ']'; + concurrencyfile << " (" << s->concurrent->size() << ")\n"; + } + } } cout << "!\n"; diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 61d55fae..708cb191 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -1809,14 +1809,6 @@ def __init__(self,s=None,graph=None,vertex=None,fmt_mask=None): if cs.route.system.devel(): continue self.route_names_and_systems.append((cs.route.list_entry_name(), cs.route.system)) - # checks for the very unusual cases where an edge ends up - # in the system as itself and its "reverse" - for e in self.vertex1.incident_s_edges: - if e.vertex1 == self.vertex2 and e.vertex2 == self.vertex1: - return - for e in self.vertex2.incident_s_edges: - if e.vertex1 == self.vertex2 and e.vertex2 == self.vertex1: - return self.vertex1.incident_s_edges.append(self) self.vertex2.incident_s_edges.append(self) self.vertex1.incident_c_edges.append(self) @@ -3220,32 +3212,27 @@ def run(self): print(".",end="",flush=True) for r in h.route_list: for s in r.segment_list: - if s.waypoint1.colocated is not None and s.waypoint2.colocated is not None: + if s.concurrent is None and s.waypoint1.colocated is not None and s.waypoint2.colocated is not None: for w1 in s.waypoint1.colocated: - if w1.route is not r: - for w2 in s.waypoint2.colocated: - if w1.route is w2.route: - other = w1.route.find_segment_by_waypoints(w1,w2) - if other is not None: - if s.concurrent is None: - s.concurrent = [] - other.concurrent = s.concurrent - s.concurrent.append(s) - s.concurrent.append(other) - concurrencyfile.write("New concurrency [" + str(s) + "][" + str(other) + "] (" + str(len(s.concurrent)) + ")\n") - else: - other.concurrent = s.concurrent - if other not in s.concurrent: - s.concurrent.append(other) - #concurrencyfile.write("Added concurrency [" + str(s) + "]-[" + str(other) + "] ("+ str(len(s.concurrent)) + ")\n") - concurrencyfile.write("Extended concurrency ") - for x in s.concurrent: - concurrencyfile.write("[" + str(x) + "]") - concurrencyfile.write(" (" + str(len(s.concurrent)) + ")\n") + for w2 in s.waypoint2.colocated: + if w1.route is w2.route and (w1 != s.waypoint1 or w2 != s.waypoint2) and (w1 != s.waypoint2 and w2 != s.waypoint1): + other = w1.route.find_segment_by_waypoints(w1,w2) + if other is not None: + if s.concurrent is None: + s.concurrent = [s] + other.concurrent = s.concurrent + s.concurrent.append(other) + concurrencyfile.write("New concurrency [" + str(s) + "][" + str(other) + "] (2)\n") + else: + other.concurrent = s.concurrent + s.concurrent.append(other) + concurrencyfile.write("Extended concurrency ") + for x in s.concurrent: + concurrencyfile.write("[" + str(x) + "]") + concurrencyfile.write(" (" + str(len(s.concurrent)) + ")\n") print("!") # now augment any traveler clinched segments for concurrencies - print(et.et() + "Augmenting travelers for detected concurrent segments.",end="",flush=True) for t in traveler_lists: print(".",end="",flush=True) From e1c26065490918d9d0f4efa94a5aa8afbdcae113 Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 19 Jul 2021 18:53:39 -0400 Subject: [PATCH 04/10] Operation Valgrind 3.10 Datacheck: delete fp before invalidating iterator main: prevent improper line.back() on empty nmpfps --- siteupdate/cplusplus/classes/Datacheck/Datacheck.cpp | 2 +- siteupdate/cplusplus/siteupdate.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/siteupdate/cplusplus/classes/Datacheck/Datacheck.cpp b/siteupdate/cplusplus/classes/Datacheck/Datacheck.cpp index 7089feb6..737902a7 100644 --- a/siteupdate/cplusplus/classes/Datacheck/Datacheck.cpp +++ b/siteupdate/cplusplus/classes/Datacheck/Datacheck.cpp @@ -88,8 +88,8 @@ void Datacheck::mark_fps(std::string& path, ElapsedTime &et) { //std::cout << "Match!" << std::endl; d.fp = 1; fpcount++; - fps.erase(fp); delete[] *fp; + fps.erase(fp); break; } else diff --git a/siteupdate/cplusplus/siteupdate.cpp b/siteupdate/cplusplus/siteupdate.cpp index 49bc9008..93567d88 100644 --- a/siteupdate/cplusplus/siteupdate.cpp +++ b/siteupdate/cplusplus/siteupdate.cpp @@ -269,7 +269,8 @@ int main(int argc, char *argv[]) unordered_set nmpfps; file.open(Args::highwaydatapath+"/nmpfps.log"); while (getline(file, line)) - { while (line.back() == 0x0D || line.back() == ' ') line.erase(line.end()-1); // trim DOS newlines & whitespace + { while (line.size() && (line.back() == 0x0D || line.back() == ' ')) + line.pop_back(); // trim DOS newlines & whitespace if (line.size()) nmpfps.insert(line); } file.close(); From e391f00be2b7b59a5d1d70ba8572362f08c9daca Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Mon, 19 Jul 2021 19:13:36 -0400 Subject: [PATCH 05/10] Operation Valgrind 3.2 TravelerList: don't "strip whitespace" before file BOF ElapsedTime: plug memory leak on internal str buffer cherrypick 827a22524eb75690d1c4da0f80bef8600168eb7c --- siteupdate/cplusplus/classes/ElapsedTime/ElapsedTime.cpp | 5 +++++ siteupdate/cplusplus/classes/ElapsedTime/ElapsedTime.h | 1 + .../cplusplus/classes/TravelerList/TravelerList.cpp | 9 +++++---- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/siteupdate/cplusplus/classes/ElapsedTime/ElapsedTime.cpp b/siteupdate/cplusplus/classes/ElapsedTime/ElapsedTime.cpp index 5a423abc..38ac83c0 100644 --- a/siteupdate/cplusplus/classes/ElapsedTime/ElapsedTime.cpp +++ b/siteupdate/cplusplus/classes/ElapsedTime/ElapsedTime.cpp @@ -5,6 +5,7 @@ ElapsedTime::ElapsedTime(int precision) format = "[%.1f] "; format[3] = '0' + precision; str = new char[15+precision]; + // deleted by ~ElapsedTime } std::string ElapsedTime::et() @@ -13,3 +14,7 @@ std::string ElapsedTime::et() sprintf(str, format.data(), elapsed.count()); return str; } + +ElapsedTime::~ElapsedTime() +{ delete[] str; +} diff --git a/siteupdate/cplusplus/classes/ElapsedTime/ElapsedTime.h b/siteupdate/cplusplus/classes/ElapsedTime/ElapsedTime.h index e5ca354d..9af048fd 100644 --- a/siteupdate/cplusplus/classes/ElapsedTime/ElapsedTime.h +++ b/siteupdate/cplusplus/classes/ElapsedTime/ElapsedTime.h @@ -9,5 +9,6 @@ class ElapsedTime public: ElapsedTime(int); + ~ElapsedTime(); std::string et(); }; diff --git a/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp b/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp index 7f0ab02c..05952d37 100644 --- a/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp +++ b/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp @@ -96,11 +96,12 @@ TravelerList::TravelerList(std::string travname, std::string* updarr[], ErrorLis // strip whitespace while (lines[l][0] == ' ' || lines[l][0] == '\t') lines[l]++; char * endchar = lines[l+1]-2; // -2 skips over the 0 inserted while separating listdata into lines - while (*endchar == 0 && endchar > lines[l]) endchar--; // skip back more for CRLF cases, and lines followed by blank lines - while (*endchar == ' ' || *endchar == '\t') - { *endchar = 0; + while (endchar > lines[l] && *endchar == 0) endchar--; // skip back more for CRLF cases, and lines followed by blank lines + if (endchar > lines[l]) + while (*endchar == ' ' || *endchar == '\t') + { *endchar = 0; endchar--; - } + } std::string trim_line(lines[l]); // ignore empty or "comment" lines if (lines[l][0] == 0 || lines[l][0] == '#') From 22747d5665ae7bf5bbf22870982ee8b9a3594a0b Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Mon, 19 Jul 2021 19:42:19 -0400 Subject: [PATCH 06/10] Operation Valgrind 3.3 comments on processing lines --- siteupdate/cplusplus/classes/Route/read_wpt.cpp | 2 ++ siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp | 1 + 2 files changed, 3 insertions(+) diff --git a/siteupdate/cplusplus/classes/Route/read_wpt.cpp b/siteupdate/cplusplus/classes/Route/read_wpt.cpp index 6f112301..efab7dda 100644 --- a/siteupdate/cplusplus/classes/Route/read_wpt.cpp +++ b/siteupdate/cplusplus/classes/Route/read_wpt.cpp @@ -34,6 +34,7 @@ void Route::read_wpt(WaypointQuadtree *all_waypoints, ErrorList *el, bool usa_fl unsigned long wptdatasize = file.tellg(); file.seekg(0, std::ios::beg); char *wptdata = new char[wptdatasize+1]; + // deleted after processing lines file.read(wptdata, wptdatasize); wptdata[wptdatasize] = 0; // add null terminator file.close(); @@ -47,6 +48,7 @@ void Route::read_wpt(WaypointQuadtree *all_waypoints, ErrorList *el, bool usa_fl } lines.push_back(wptdata+wptdatasize+1); // add a dummy "past-the-end" element to make lines[l+1]-2 work + // process lines for (unsigned int l = 0; l < lines.size()-1; l++) { // strip whitespace from end... char* endchar = lines[l+1]-2; // -2 skips over the 0 inserted while splitting wptdata into lines diff --git a/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp b/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp index 05952d37..0cb1c44b 100644 --- a/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp +++ b/siteupdate/cplusplus/classes/TravelerList/TravelerList.cpp @@ -55,6 +55,7 @@ TravelerList::TravelerList(std::string travname, std::string* updarr[], ErrorLis unsigned long listdatasize = file.tellg(); file.seekg(0, std::ios::beg); char *listdata = new char[listdatasize+1]; + // deleted after processing lines file.read(listdata, listdatasize); listdata[listdatasize] = 0; // add null terminator file.close(); From 87c17de4a742013b62cee0e7ed3e337b05402b9f Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Mon, 19 Jul 2021 20:52:07 -0400 Subject: [PATCH 07/10] Operation Valgrind 3.45 delete unmatchedfps --- siteupdate/cplusplus/classes/Datacheck/Datacheck.cpp | 8 +++++--- siteupdate/python-teresco/siteupdate.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/siteupdate/cplusplus/classes/Datacheck/Datacheck.cpp b/siteupdate/cplusplus/classes/Datacheck/Datacheck.cpp index 737902a7..d7f6bf9c 100644 --- a/siteupdate/cplusplus/classes/Datacheck/Datacheck.cpp +++ b/siteupdate/cplusplus/classes/Datacheck/Datacheck.cpp @@ -56,7 +56,7 @@ void Datacheck::read_fps(std::string& path, ErrorList &el) // parse datacheckfps.csv line size_t NumFields = 6; std::string* fields = new std::string[6]; - // deleted when FP is matched or on termination of program + // deleted when FP is matched or when writing unmatchedfps.log std::string* ptr_array[6] = {&fields[0], &fields[1], &fields[2], &fields[3], &fields[4], &fields[5]}; split(line, ptr_array, NumFields, ';'); if (NumFields != 6) @@ -103,13 +103,15 @@ void Datacheck::mark_fps(std::string& path, ElapsedTime &et) } void Datacheck::unmatchedfps_log(std::string& path) -{ // write log of unmatched false positives from the datacheckfps.csv +{ // write log of unmatched false positives from datacheckfps.csv std::ofstream fpfile(path+"/unmatchedfps.log"); time_t timestamp = time(0); fpfile << "Log file created at: " << ctime(×tamp); if (fps.empty()) fpfile << "No unmatched FP entries.\n"; else for (std::string* entry : fps) - fpfile << entry[0] << ';' << entry[1] << ';' << entry[2] << ';' << entry[3] << ';' << entry[4] << ';' << entry[5] << '\n'; + { fpfile << entry[0] << ';' << entry[1] << ';' << entry[2] << ';' << entry[3] << ';' << entry[4] << ';' << entry[5] << '\n'; + delete[] entry; + } fpfile.close(); } diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 708cb191..2b6640b6 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -4157,7 +4157,7 @@ def run(self): print("!", flush=True) print(et.et() + "Found " + str(len(datacheckerrors)) + " datacheck errors and matched " + str(fpcount) + " FP entries.", flush=True) -# write log of unmatched false positives from the datacheckfps.csv +# write log of unmatched false positives from datacheckfps.csv print(et.et() + "Writing log of unmatched datacheck FP entries.", flush=True) fpfile = open(args.logfilepath+'/unmatchedfps.log','w',encoding='utf-8') fpfile.write("Log file created at: " + str(datetime.datetime.now()) + "\n") From e259ec3d4d455508969944ee2746ca6c9a539f19 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Mon, 19 Jul 2021 20:55:22 -0400 Subject: [PATCH 08/10] Operation Valgrind 3.54 Plug memory leaks in subgraph region/system lists cherrypick d5810b8d9c1e757183097163e77f3e6783682953 --- siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp | 4 +++- siteupdate/cplusplus/tasks/subgraphs/continent.cpp | 2 +- siteupdate/cplusplus/tasks/subgraphs/country.cpp | 2 +- siteupdate/cplusplus/tasks/subgraphs/multiregion.cpp | 2 +- siteupdate/cplusplus/tasks/subgraphs/multisystem.cpp | 2 +- siteupdate/cplusplus/tasks/subgraphs/region.cpp | 2 +- siteupdate/cplusplus/tasks/subgraphs/system.cpp | 2 +- 7 files changed, 9 insertions(+), 7 deletions(-) diff --git a/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp b/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp index 2f8cbbd1..56c28be8 100644 --- a/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp +++ b/siteupdate/cplusplus/classes/GraphGeneration/HighwayGraph.cpp @@ -401,7 +401,7 @@ void HighwayGraph::write_subgraphs_tmg } #ifdef threading_enabled term->lock(); - if (GRAPH(0).cat != GraphListEntry::entries[graphnum-1].cat) + if (GRAPH(0).cat != GRAPH(-1).cat) std::cout << '\n' << et->et() << "Writing " << GRAPH(0).category() << " graphs.\n"; #endif std::cout << GRAPH(0).tag() @@ -462,6 +462,8 @@ void HighwayGraph::write_subgraphs_tmg simplefile.close(); collapfile.close(); travelfile.close(); + if (GRAPH(0).regions) delete GRAPH(0).regions; + if (GRAPH(0).systems) delete GRAPH(0).systems; GRAPH(0).vertices = mv.size(); GRAPH(0).edges = mse.size(); GRAPH(0).travelers = 0; GRAPH(1).vertices = cv_count; GRAPH(1).edges = mce.size(); GRAPH(1).travelers = 0; diff --git a/siteupdate/cplusplus/tasks/subgraphs/continent.cpp b/siteupdate/cplusplus/tasks/subgraphs/continent.cpp index 4a9af9f2..401bb4f3 100644 --- a/siteupdate/cplusplus/tasks/subgraphs/continent.cpp +++ b/siteupdate/cplusplus/tasks/subgraphs/continent.cpp @@ -8,7 +8,7 @@ cout << et.et() << "Creating continent graphs." << endl; // add entries to graph vector for (size_t c = 0; c < continents.size()-1; c++) { regions = new list; - // deleted on termination of program + // deleted @ end of HighwayGraph::write_subgraphs_tmg for (Region* r : Region::allregions) // does it match this continent and have routes? if (&continents[c] == r->continent && r->active_preview_mileage) diff --git a/siteupdate/cplusplus/tasks/subgraphs/country.cpp b/siteupdate/cplusplus/tasks/subgraphs/country.cpp index b68817d5..0328a514 100644 --- a/siteupdate/cplusplus/tasks/subgraphs/country.cpp +++ b/siteupdate/cplusplus/tasks/subgraphs/country.cpp @@ -8,7 +8,7 @@ cout << et.et() << "Creating country graphs." << endl; // add entries to graph vector for (size_t c = 0; c < countries.size()-1; c++) { regions = new list; - // deleted on termination of program + // deleted @ end of HighwayGraph::write_subgraphs_tmg for (Region* r : Region::allregions) // does it match this country and have routes? if (&countries[c] == r->country && r->active_preview_mileage) diff --git a/siteupdate/cplusplus/tasks/subgraphs/multiregion.cpp b/siteupdate/cplusplus/tasks/subgraphs/multiregion.cpp index be8e8967..5ab091ae 100644 --- a/siteupdate/cplusplus/tasks/subgraphs/multiregion.cpp +++ b/siteupdate/cplusplus/tasks/subgraphs/multiregion.cpp @@ -27,7 +27,7 @@ while (getline(file, line)) el.add_error("title > " + std::to_string(DBFieldLength::graphFilename-14) + " bytes in multiregion.csv line: " + line); regions = new list; - // deleted on termination of program + // deleted @ end of HighwayGraph::write_subgraphs_tmg for(char* rg = strtok(fields[2], ","); rg; rg = strtok(0, ",")) for (Region* r : Region::allregions) if (rg == r->code) diff --git a/siteupdate/cplusplus/tasks/subgraphs/multisystem.cpp b/siteupdate/cplusplus/tasks/subgraphs/multisystem.cpp index 4daacab6..524a92be 100644 --- a/siteupdate/cplusplus/tasks/subgraphs/multisystem.cpp +++ b/siteupdate/cplusplus/tasks/subgraphs/multisystem.cpp @@ -26,7 +26,7 @@ while (getline(file, line)) el.add_error("title > " + std::to_string(DBFieldLength::graphFilename-14) + " bytes in multisystem.csv line: " + line); systems = new list; - // deleted on termination of program + // deleted @ end of HighwayGraph::write_subgraphs_tmg for(char* s = strtok(fields[2], ","); s; s = strtok(0, ",")) for (HighwaySystem *h : HighwaySystem::syslist) if (s == h->systemname) diff --git a/siteupdate/cplusplus/tasks/subgraphs/region.cpp b/siteupdate/cplusplus/tasks/subgraphs/region.cpp index 9bd3c8ec..a9216563 100644 --- a/siteupdate/cplusplus/tasks/subgraphs/region.cpp +++ b/siteupdate/cplusplus/tasks/subgraphs/region.cpp @@ -11,7 +11,7 @@ cout << et.et() << "Creating regional data graphs." << endl; for (Region* region : Region::allregions) { if (region->active_preview_mileage == 0) continue; regions = new list(1, region); - // deleted on termination of program + // deleted @ end of HighwayGraph::write_subgraphs_tmg ADDGRAPH('s'); ADDGRAPH('c'); ADDGRAPH('t'); diff --git a/siteupdate/cplusplus/tasks/subgraphs/system.cpp b/siteupdate/cplusplus/tasks/subgraphs/system.cpp index 3e977770..2aa4e062 100644 --- a/siteupdate/cplusplus/tasks/subgraphs/system.cpp +++ b/siteupdate/cplusplus/tasks/subgraphs/system.cpp @@ -20,7 +20,7 @@ while (getline(file, line)) } if (h) { systems = new list(1, h); - // deleted on termination of program + // deleted @ end of HighwayGraph::write_subgraphs_tmg ADDGRAPH('s'); ADDGRAPH('c'); ADDGRAPH('t'); From e19ec3bdd37ae71282d509b06e2e58d58e2ef7c5 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Mon, 19 Jul 2021 22:04:16 -0400 Subject: [PATCH 09/10] Operation Valgrind 3.6 delete[] updates & systemupdates --- siteupdate/cplusplus/functions/sql_file.cpp | 2 ++ siteupdate/cplusplus/tasks/read_updates.cpp | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/siteupdate/cplusplus/functions/sql_file.cpp b/siteupdate/cplusplus/functions/sql_file.cpp index 6a6db82e..0225f5e0 100644 --- a/siteupdate/cplusplus/functions/sql_file.cpp +++ b/siteupdate/cplusplus/functions/sql_file.cpp @@ -391,6 +391,7 @@ void sqlfile1 first = 0; sqlfile << "('" << update[0] << "','" << double_quotes(update[1]) << "','" << double_quotes(update[2]) << "','" << update[3] << "','" << double_quotes(update[4]) << "')\n"; + delete[] update; } sqlfile << ";\n"; @@ -411,6 +412,7 @@ void sqlfile1 first = 0; sqlfile << "('" << systemupdate[0] << "','" << double_quotes(systemupdate[1]) << "','" << systemupdate[2] << "','" << double_quotes(systemupdate[3]) << "','" << systemupdate[4] << "')\n"; + delete[] systemupdate; } sqlfile << ";\n"; sqlfile.close(); diff --git a/siteupdate/cplusplus/tasks/read_updates.cpp b/siteupdate/cplusplus/tasks/read_updates.cpp index 89a9b272..f9d2c3ff 100644 --- a/siteupdate/cplusplus/tasks/read_updates.cpp +++ b/siteupdate/cplusplus/tasks/read_updates.cpp @@ -12,7 +12,7 @@ while (getline(file, line)) // parse updates.csv line size_t NumFields = 5; string* fields = new string[5]; - // deleted on termination of program + // deleted as DB table is written string* ptr_array[5] = {&fields[0], &fields[1], &fields[2], &fields[3], &fields[4]}; split(line, ptr_array, NumFields, ';'); if (NumFields != 5) @@ -74,7 +74,7 @@ while (getline(file, line)) // parse systemupdates.csv line size_t NumFields = 5; string* fields = new string[5]; - // deleted on termination of program + // deleted as DB table is written string* ptr_array[5] = {&fields[0], &fields[1], &fields[2], &fields[3], &fields[4]}; split(line, ptr_array, NumFields, ';'); if (NumFields != 5) From 1aedacb7b2c34a0775da0ad9dd6e83d559ecbce5 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Mon, 19 Jul 2021 22:17:52 -0400 Subject: [PATCH 10/10] Operation Valgrind 3.7 avoid invalid reads in read_wpt while coping files that are empty or begin with null zero https://github.com/TravelMapping/DataProcessing/issues/248#issuecomment-877359437 --- siteupdate/cplusplus/classes/Route/read_wpt.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/siteupdate/cplusplus/classes/Route/read_wpt.cpp b/siteupdate/cplusplus/classes/Route/read_wpt.cpp index efab7dda..2f916c8e 100644 --- a/siteupdate/cplusplus/classes/Route/read_wpt.cpp +++ b/siteupdate/cplusplus/classes/Route/read_wpt.cpp @@ -46,10 +46,15 @@ void Route::read_wpt(WaypointQuadtree *all_waypoints, ErrorList *el, bool usa_fl { for (spn = strcspn(c, "\n\r"); c[spn] == '\n' || c[spn] == '\r'; spn++) c[spn] = 0; lines.emplace_back(c); } + if (lines.empty()) + { delete[] wptdata; + el->add_error(filename + " is empty or begins with null zero"); + return; + } lines.push_back(wptdata+wptdatasize+1); // add a dummy "past-the-end" element to make lines[l+1]-2 work // process lines - for (unsigned int l = 0; l < lines.size()-1; l++) + for (unsigned int l = lines[1] < wptdata+2; l < lines.size()-1; l++) { // strip whitespace from end... char* endchar = lines[l+1]-2; // -2 skips over the 0 inserted while splitting wptdata into lines while (*endchar == 0) endchar--; // skip back more for CRLF cases, and lines followed by blank lines