From 0ec9b8e41d79cd680a8cbafb7e60fc04ab45c9af Mon Sep 17 00:00:00 2001 From: Daniel Spitzbart Date: Thu, 17 Jan 2019 16:02:06 +0100 Subject: [PATCH 1/6] making overwrite for nanoAODFromDAS more intelligent --- core/python/Sample.py | 52 +++++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/core/python/Sample.py b/core/python/Sample.py index 09bd03a..e602717 100644 --- a/core/python/Sample.py +++ b/core/python/Sample.py @@ -286,16 +286,20 @@ def nanoAODfromDAS(cls, name, DASname, instance = 'global', redirector='root://h else: cache = None + # first check if there are already files in the cache + if n_cache_files: + filesFromCache = [ f["value"] for f in cache.getDicts({'name':name, 'DAS':DASname}) ] + else: + filesFromCache = [] + # if we don't want to overwrite, and there's a filelist in the cache we're already done if n_cache_files and not overwrite: - files = [ f["value"] for f in cache.getDicts({'name':name, 'DAS':DASname}) ] + files = filesFromCache normalization = cache.getDicts({'name':name, 'DAS':DASname})[0]["normalization"] logger.info('Found sample %s in cache %s, return %i files.', name, dbFile, len(files)) else: - if overwrite: - cache.removeObjects({"name":name, 'DAS':DASname}) def _dasPopen(dbs): if 'LSB_JOBID' in os.environ: @@ -317,23 +321,33 @@ def _dasPopen(dbs): filename = redirector+'/'+line files.append(filename) - if DASname.endswith('SIM') or not 'Run20' in DASname: - # need to read the proper normalization for MC - logger.info("Reading normalization. This is slow, so grab a coffee.") - tmp_sample = cls(name=name, files=files, treeName = treeName, selectionString = selectionString, weightString = weightString, - isData = isData, color=color, texName = texName, xSection = xSection, normalization=1) - normalization = tmp_sample.getYieldFromDraw('(1)', genWeight)['val'] - logger.info("Got normalization %s", normalization) + if sorted(files) == sorted(filesFromCache): + # if the files didn't change we don't need to read the normalization again (slowest part!) + logger.info("File list didn't change. Skipping.") + normalization = cache.getDicts({'name':name, 'DAS':DASname})[0]["normalization"] + else: - # for data, we can just use the number of events, although no normalization is needed anyway. - dbs='dasgoclient -query="summary %s=%s instance=prod/%s" --format=json'%(qwhat,query, instance) - jdata = json.load(_dasPopen(dbs))['data'][0]['summary'][0] - normalization = int(jdata['nevents']) - - if overwrite or n_cache_files<1: - for f in files: - if cache is not None: - cache.add({"name":name, 'DAS':DASname, 'normalization':str(normalization)}, f, save=True) + if overwrite: + # remove old entry + cache.removeObjects({"name":name, 'DAS':DASname}) + + if DASname.endswith('SIM') or not 'Run20' in DASname: + # need to read the proper normalization for MC + logger.info("Reading normalization. This is slow, so grab a coffee.") + tmp_sample = cls(name=name, files=files, treeName = treeName, selectionString = selectionString, weightString = weightString, + isData = isData, color=color, texName = texName, xSection = xSection, normalization=1) + normalization = tmp_sample.getYieldFromDraw('(1)', genWeight)['val'] + logger.info("Got normalization %s", normalization) + else: + # for data, we can just use the number of events, although no normalization is needed anyway. + dbs='dasgoclient -query="summary %s=%s instance=prod/%s" --format=json'%(qwhat,query, instance) + jdata = json.load(_dasPopen(dbs))['data'][0]['summary'][0] + normalization = int(jdata['nevents']) + +# if overwrite or n_cache_files<1: + for f in files: + if cache is not None: + cache.add({"name":name, 'DAS':DASname, 'normalization':str(normalization)}, f, save=True) if limit>0: files=files[:limit] sample = cls(name=name, files=files, treeName = treeName, selectionString = selectionString, weightString = weightString, From 23eb5f648c92ff6dd5eef3beaff940b172831f48 Mon Sep 17 00:00:00 2001 From: Daniel Spitzbart Date: Thu, 17 Jan 2019 16:06:40 +0100 Subject: [PATCH 2/6] minor clean-up --- core/python/Sample.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/python/Sample.py b/core/python/Sample.py index e602717..168e044 100644 --- a/core/python/Sample.py +++ b/core/python/Sample.py @@ -344,7 +344,6 @@ def _dasPopen(dbs): jdata = json.load(_dasPopen(dbs))['data'][0]['summary'][0] normalization = int(jdata['nevents']) -# if overwrite or n_cache_files<1: for f in files: if cache is not None: cache.add({"name":name, 'DAS':DASname, 'normalization':str(normalization)}, f, save=True) From a6022562035189ceb95e8e7b152202683dad7346 Mon Sep 17 00:00:00 2001 From: Daniel Spitzbart Date: Thu, 17 Jan 2019 16:27:58 +0100 Subject: [PATCH 3/6] cosmetics --- core/python/Sample.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/python/Sample.py b/core/python/Sample.py index 168e044..56db472 100644 --- a/core/python/Sample.py +++ b/core/python/Sample.py @@ -325,11 +325,13 @@ def _dasPopen(dbs): # if the files didn't change we don't need to read the normalization again (slowest part!) logger.info("File list didn't change. Skipping.") normalization = cache.getDicts({'name':name, 'DAS':DASname})[0]["normalization"] + logger.info('Sample %s from cache %s returned %i files.', name, dbFile, len(files)) else: if overwrite: # remove old entry cache.removeObjects({"name":name, 'DAS':DASname}) + logger.info("Removed old DB entry.") if DASname.endswith('SIM') or not 'Run20' in DASname: # need to read the proper normalization for MC @@ -347,6 +349,8 @@ def _dasPopen(dbs): for f in files: if cache is not None: cache.add({"name":name, 'DAS':DASname, 'normalization':str(normalization)}, f, save=True) + + logger.info('Found sample %s in cache %s, return %i files.', name, dbFile, len(files)) if limit>0: files=files[:limit] sample = cls(name=name, files=files, treeName = treeName, selectionString = selectionString, weightString = weightString, From 8e7652507b0baa5932bec257a15238d6f0ab6c5a Mon Sep 17 00:00:00 2001 From: Daniel Spitzbart Date: Thu, 17 Jan 2019 16:42:57 +0100 Subject: [PATCH 4/6] also check the normalization --- core/python/Sample.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/core/python/Sample.py b/core/python/Sample.py index 56db472..080ce73 100644 --- a/core/python/Sample.py +++ b/core/python/Sample.py @@ -288,14 +288,15 @@ def nanoAODfromDAS(cls, name, DASname, instance = 'global', redirector='root://h # first check if there are already files in the cache if n_cache_files: - filesFromCache = [ f["value"] for f in cache.getDicts({'name':name, 'DAS':DASname}) ] + filesFromCache = [ f["value"] for f in cache.getDicts({'name':name, 'DAS':DASname}) ] + normalizationFromCache = cache.getDicts({'name':name, 'DAS':DASname})[0]["normalization"] else: filesFromCache = [] # if we don't want to overwrite, and there's a filelist in the cache we're already done if n_cache_files and not overwrite: - files = filesFromCache - normalization = cache.getDicts({'name':name, 'DAS':DASname})[0]["normalization"] + files = filesFromCache + normalization = normalizationFromCache logger.info('Found sample %s in cache %s, return %i files.', name, dbFile, len(files)) @@ -321,10 +322,10 @@ def _dasPopen(dbs): filename = redirector+'/'+line files.append(filename) - if sorted(files) == sorted(filesFromCache): - # if the files didn't change we don't need to read the normalization again (slowest part!) + if sorted(files) == sorted(filesFromCache) and normalizationFromCache>0: + # if the files didn't change we don't need to read the normalization again (slowest part!). If the norm was 0 previously, also get it again. logger.info("File list didn't change. Skipping.") - normalization = cache.getDicts({'name':name, 'DAS':DASname})[0]["normalization"] + normalization = normalizationFromCache logger.info('Sample %s from cache %s returned %i files.', name, dbFile, len(files)) else: From 4287b158eddd37825c2b00054edff3da8ff5b895 Mon Sep 17 00:00:00 2001 From: Daniel Spitzbart Date: Fri, 18 Jan 2019 12:13:58 +0100 Subject: [PATCH 5/6] NanoAOD: only try to read a normalization if files are available on local T2 --- core/python/Sample.py | 42 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/core/python/Sample.py b/core/python/Sample.py index 080ce73..f04613b 100644 --- a/core/python/Sample.py +++ b/core/python/Sample.py @@ -269,7 +269,7 @@ def fromDirectory(cls, name, directory, treeName = "Events", normalization = Non @classmethod def nanoAODfromDAS(cls, name, DASname, instance = 'global', redirector='root://hephyse.oeaw.ac.at/', dbFile=None, overwrite=False, treeName = "Events", maxN = None, \ selectionString = None, weightString = None, xSection=-1, - isData = False, color = 0, texName = None, multithreading=True, genWeight='genWeight', json=None): + isData = False, color = 0, texName = None, multithreading=True, genWeight='genWeight', json=None, localSite='T2_AT_Vienna'): ''' get nanoAOD from DAS and make a local copy on afs ''' @@ -322,9 +322,9 @@ def _dasPopen(dbs): filename = redirector+'/'+line files.append(filename) - if sorted(files) == sorted(filesFromCache) and normalizationFromCache>0: + if (sorted(files) == sorted(filesFromCache)) and float(normalizationFromCache) > 0.0: # if the files didn't change we don't need to read the normalization again (slowest part!). If the norm was 0 previously, also get it again. - logger.info("File list didn't change. Skipping.") + logger.info("File list for %s didn't change. Skipping.", name) normalization = normalizationFromCache logger.info('Sample %s from cache %s returned %i files.', name, dbFile, len(files)) @@ -334,13 +334,38 @@ def _dasPopen(dbs): cache.removeObjects({"name":name, 'DAS':DASname}) logger.info("Removed old DB entry.") + if instance == 'global': + # check if dataset is available in local site, otherwise don't read a normalization + dbs='dasgoclient -query="site %s=%s instance=prod/%s" --format=json'%(qwhat,query, instance) + jdata = json.load(_dasPopen(dbs)) + + filesOnLocalT2 = False + for d in jdata['data']: + if d['site'][0]['name'] == localSite and d['site'][0].has_key('replica_fraction'): + fraction = d['site'][0]['replica_fraction'] + if float(str(fraction).replace('%','')) < 100.: + filesOnLocalT2 = False + break + else: + filesOnLocalT2 = True + else: + # if we produced the samples ourselves we don't need to check this + filesOnLocalT2 = True + + if filesOnLocalT2: + logger.info("Files are available at %s", localSite) + if DASname.endswith('SIM') or not 'Run20' in DASname: # need to read the proper normalization for MC - logger.info("Reading normalization. This is slow, so grab a coffee.") - tmp_sample = cls(name=name, files=files, treeName = treeName, selectionString = selectionString, weightString = weightString, - isData = isData, color=color, texName = texName, xSection = xSection, normalization=1) - normalization = tmp_sample.getYieldFromDraw('(1)', genWeight)['val'] - logger.info("Got normalization %s", normalization) + if filesOnLocalT2: + logger.info("Reading normalization. This is slow, so grab a coffee.") + tmp_sample = cls(name=name, files=files, treeName = treeName, selectionString = selectionString, weightString = weightString, + isData = isData, color=color, texName = texName, xSection = xSection, normalization=1) + normalization = tmp_sample.getYieldFromDraw('(1)', genWeight)['val'] + logger.info("Got normalization %s", normalization) + else: + logger.info("Files only available on the grid. Can't read a normalization.") + normalization = 0 else: # for data, we can just use the number of events, although no normalization is needed anyway. dbs='dasgoclient -query="summary %s=%s instance=prod/%s" --format=json'%(qwhat,query, instance) @@ -352,6 +377,7 @@ def _dasPopen(dbs): cache.add({"name":name, 'DAS':DASname, 'normalization':str(normalization)}, f, save=True) logger.info('Found sample %s in cache %s, return %i files.', name, dbFile, len(files)) + if limit>0: files=files[:limit] sample = cls(name=name, files=files, treeName = treeName, selectionString = selectionString, weightString = weightString, From 885bd115683b10a8c2dcef370d0e7e121606d1ec Mon Sep 17 00:00:00 2001 From: Daniel Spitzbart Date: Tue, 29 Jan 2019 14:47:46 +0100 Subject: [PATCH 6/6] fix, add TGraph for drawObjects --- core/python/Sample.py | 1 + plot/python/plotting.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/core/python/Sample.py b/core/python/Sample.py index f04613b..abc2a58 100644 --- a/core/python/Sample.py +++ b/core/python/Sample.py @@ -287,6 +287,7 @@ def nanoAODfromDAS(cls, name, DASname, instance = 'global', redirector='root://h cache = None # first check if there are already files in the cache + normalizationFromCache = 0. if n_cache_files: filesFromCache = [ f["value"] for f in cache.getDicts({'name':name, 'DAS':DASname}) ] normalizationFromCache = cache.getDicts({'name':name, 'DAS':DASname})[0]["normalization"] diff --git a/plot/python/plotting.py b/plot/python/plotting.py index 59caeb6..5d2d4ce 100644 --- a/plot/python/plotting.py +++ b/plot/python/plotting.py @@ -462,7 +462,7 @@ def draw(plot, \ for o in drawObjects: if o: - if type(o) in [ ROOT.TF1 ]: + if type(o) in [ ROOT.TF1, ROOT.TGraph ]: o.Draw('same') else: o.Draw()