From 89916788b9a96847247d8170f8b3233639383622 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Mon, 7 Jan 2019 00:05:51 -0500 Subject: [PATCH 1/2] move fatal error reporting before graph generation --- siteupdate/python-teresco/siteupdate.py | 55 +++++++++++++------------ 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index 545e96eb..bdbb6cb5 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2860,6 +2860,34 @@ def run(self): sysfile.write('\n') sysfile.close() +# read in the datacheck false positives list +print(et.et() + "Reading datacheckfps.csv.",flush=True) +with open(args.highwaydatapath+"/datacheckfps.csv", "rt",encoding='utf-8') as file: + lines = file.readlines() + +lines.pop(0) # ignore header line +datacheckfps = [] +datacheck_always_error = [ 'DUPLICATE_LABEL', 'HIDDEN_TERMINUS', + 'LABEL_INVALID_CHAR', 'LABEL_SLASHES', + 'LONG_UNDERSCORE', 'MALFORMED_URL', + 'NONTERMINAL_UNDERSCORE' ] +for line in lines: + fields = line.rstrip('\n').split(';') + if len(fields) != 6: + el.add_error("Could not parse datacheckfps.csv line: " + line) + continue + if fields[4] in datacheck_always_error: + print("datacheckfps.csv line not allowed (always error): " + line) + continue + datacheckfps.append(fields) + +# See if we have any errors that should be fatal to the site update process +if len(el.error_list) > 0: + print("ABORTING due to " + str(len(el.error_list)) + " errors:") + for i in range(len(el.error_list)): + print(str(i+1) + ": " + el.error_list[i]) + sys.exit(1) + # Build a graph structure out of all highway data in active and # preview systems print(et.et() + "Setting up for graphs of highway data.", flush=True) @@ -3051,33 +3079,6 @@ def run(self): # data check: visit each system and route and check for various problems print(et.et() + "Performing data checks.",end="",flush=True) -# first, read in the false positives list -with open(args.highwaydatapath+"/datacheckfps.csv", "rt",encoding='utf-8') as file: - lines = file.readlines() - -lines.pop(0) # ignore header line -datacheckfps = [] -datacheck_always_error = [ 'DUPLICATE_LABEL', 'HIDDEN_TERMINUS', - 'LABEL_INVALID_CHAR', 'LABEL_SLASHES', - 'LONG_UNDERSCORE', 'MALFORMED_URL', - 'NONTERMINAL_UNDERSCORE' ] -for line in lines: - fields = line.rstrip('\n').split(';') - if len(fields) != 6: - el.add_error("Could not parse datacheckfps.csv line: " + line) - continue - if fields[4] in datacheck_always_error: - print("datacheckfps.csv line not allowed (always error): " + line) - continue - datacheckfps.append(fields) - -# See if we have any errors that should be fatal to the site update process -if len(el.error_list) > 0: - print("ABORTING due to " + str(len(el.error_list)) + " errors:") - for i in range(len(el.error_list)): - print(str(i+1) + ": " + el.error_list[i]) - sys.exit(1) - # perform most datachecks here (list initialized above) for h in highway_systems: print(".",end="",flush=True) From 5db8dfa9366dcc34bc7862041cd80fc3d86513b3 Mon Sep 17 00:00:00 2001 From: Eric Bryant Date: Mon, 7 Jan 2019 11:41:35 -0500 Subject: [PATCH 2/2] add missing file.close() --- siteupdate/python-teresco/siteupdate.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/siteupdate/python-teresco/siteupdate.py b/siteupdate/python-teresco/siteupdate.py index bdbb6cb5..85f4ab54 100755 --- a/siteupdate/python-teresco/siteupdate.py +++ b/siteupdate/python-teresco/siteupdate.py @@ -2359,6 +2359,7 @@ def run(self): print(et.et() + "Reading updates file. ",end="",flush=True) with open(args.highwaydatapath+"/updates.csv", "rt", encoding='UTF-8') as file: lines = file.readlines() +file.close() lines.pop(0) # ignore header line for line in lines: @@ -2376,6 +2377,7 @@ def run(self): print(et.et() + "Reading systemupdates file. ",end="",flush=True) with open(args.highwaydatapath+"/systemupdates.csv", "rt", encoding='UTF-8') as file: lines = file.readlines() +file.close() lines.pop(0) # ignore header line for line in lines: @@ -2864,6 +2866,7 @@ def run(self): print(et.et() + "Reading datacheckfps.csv.",flush=True) with open(args.highwaydatapath+"/datacheckfps.csv", "rt",encoding='utf-8') as file: lines = file.readlines() +file.close() lines.pop(0) # ignore header line datacheckfps = [] @@ -2922,6 +2925,7 @@ def run(self): print("\n" + et.et() + "Creating area data graphs.", flush=True) with open(args.highwaydatapath+"/graphs/areagraphs.csv", "rt",encoding='utf-8') as file: lines = file.readlines() + file.close() lines.pop(0); # ignore header line area_list = [] for line in lines: @@ -2969,6 +2973,7 @@ def run(self): h = None with open(args.highwaydatapath+"/graphs/systemgraphs.csv", "rt",encoding='utf-8') as file: lines = file.readlines() + file.close() lines.pop(0); # ignore header line for hname in lines: h = None @@ -2992,6 +2997,7 @@ def run(self): with open(args.highwaydatapath+"/graphs/multisystem.csv", "rt",encoding='utf-8') as file: lines = file.readlines() + file.close() lines.pop(0); # ignore header line for line in lines: fields = line.rstrip('\n').split(";") @@ -3017,6 +3023,7 @@ def run(self): with open(args.highwaydatapath+"/graphs/multiregion.csv", "rt",encoding='utf-8') as file: lines = file.readlines() + file.close() lines.pop(0); # ignore header line for line in lines: fields = line.rstrip('\n').split(";")