From e5bb3d60c3c9b2bf6252bf5ad28fae75707090cf Mon Sep 17 00:00:00 2001 From: Robert Schoefbeck Date: Thu, 2 Mar 2017 20:33:19 +0100 Subject: [PATCH 01/16] fix clone tree --- core/python/TreeReader.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/core/python/TreeReader.py b/core/python/TreeReader.py index 7fecf9e..be4822b 100644 --- a/core/python/TreeReader.py +++ b/core/python/TreeReader.py @@ -114,7 +114,12 @@ def cloneTree(self, branchList = [], newTreename = None, rootfile = None): # Copying tree # 1. causes trouble with few events and few files. - #res = self.sample.chain.CopyTree( "(1)", "" ) + # res = self.sample.chain.CopyTree( "(1)", "" ) + + # 1.5: this seems to give misaligned l1_pt and LepGood_pt??? + # tree = self.sample.chain.GetTree() + # tree.SetEventList( list_to_copy ) + # res = tree.CopyTree( "(1)", "" ) # 2. Doesn't take into account event list #res = self.sample.chain.CloneTree( 0 ) @@ -122,10 +127,10 @@ def cloneTree(self, branchList = [], newTreename = None, rootfile = None): # 3. Loop (no problems observed) logger.debug("Copying %i events in a loop.", list_to_copy.GetN()) - tree = self.sample.chain.GetTree() + #tree = self.sample.chain.GetTree() res = self.sample.chain.GetTree().CloneTree( 0 ) for i_event in xrange(list_to_copy.GetN()): - tree.GetEntry( list_to_copy.GetEntry(i_event) ) + self.sample.chain.GetEntry( list_to_copy.GetEntry(i_event) ) res.Fill() # Needed? res.Write() From 0e458701f0498f02e5f8878916a352a1a47a2dbd Mon Sep 17 00:00:00 2001 From: Robert Schoefbeck Date: Wed, 8 Mar 2017 10:21:54 +0100 Subject: [PATCH 02/16] removing unused naming functionality in getEventList --- core/python/Sample.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/core/python/Sample.py b/core/python/Sample.py index 49dbb47..5e4f622 100644 --- a/core/python/Sample.py +++ b/core/python/Sample.py @@ -430,7 +430,7 @@ def __loadChain(self): else: logger.error( "Check of root file failed. Skipping. File: %s", f ) except IOError as e: - logger.warning( "Could not load file %s", f ) + logger.error( "Could not load file %s", f ) raise e logger.debug( "Loaded %i files for sample '%s'.", counter, self.name ) @@ -525,7 +525,7 @@ def combineWithSampleWeight(self, weightString): self.name, weightString ) return weightString - def getEventList(self, selectionString=None, name=None): + def getEventList(self, selectionString=None): ''' Get a TEventList from a selectionString (combined with self.selectionString, if exists). ''' @@ -536,12 +536,7 @@ def getEventList(self, selectionString=None, name=None): self.chain.Draw('>>'+tmp, selectionString_ if selectionString else "(1)") elistTMP_t = ROOT.gDirectory.Get(tmp) - if not name: - return elistTMP_t - else: - elistTMP = elistTMP_t.Clone(name) - del elistTMP_t - return elistTMP + return elistTMP_t def getYieldFromDraw(self, selectionString = None, weightString = None): ''' Get yield from self.chain according to a selectionString and a weightString From 14f3fdc729184b4400d5e8cf745b5774173558c4 Mon Sep 17 00:00:00 2001 From: Robert Schoefbeck Date: Wed, 8 Mar 2017 10:28:25 +0100 Subject: [PATCH 03/16] fix bug when running over a small number of events. More care with event lists --- core/python/LooperBase.py | 2 +- core/python/Sample.py | 2 +- core/python/TreeMaker.py | 10 +++- core/python/TreeReader.py | 111 +++++++++++++++----------------------- 4 files changed, 55 insertions(+), 70 deletions(-) diff --git a/core/python/LooperBase.py b/core/python/LooperBase.py index 25a655d..b6ca82c 100644 --- a/core/python/LooperBase.py +++ b/core/python/LooperBase.py @@ -34,7 +34,7 @@ def __init__(self, variables): # Internal state for running self.position = -1 - self.eList = None + self._eList = None self.classUUIDs = [] diff --git a/core/python/Sample.py b/core/python/Sample.py index 5e4f622..59a1e93 100644 --- a/core/python/Sample.py +++ b/core/python/Sample.py @@ -532,7 +532,7 @@ def getEventList(self, selectionString=None): selectionString_ = self.combineWithSampleSelection( selectionString ) tmp=str(uuid.uuid4()) - logger.debug( "Making eList for sample %s and selectionString %s", self.name, selectionString_ ) + logger.debug( "Making event list for sample %s and selectionString %s", self.name, selectionString_ ) self.chain.Draw('>>'+tmp, selectionString_ if selectionString else "(1)") elistTMP_t = ROOT.gDirectory.Get(tmp) diff --git a/core/python/TreeMaker.py b/core/python/TreeMaker.py index d7d9e36..7a7dc3e 100644 --- a/core/python/TreeMaker.py +++ b/core/python/TreeMaker.py @@ -33,6 +33,12 @@ def __init__(self, variables, sequence = [], treeName = "Events"): # function to fill the event self.sequence = sequence + def debugBranchAddresses( self, prefix = ""): + ''' If strings are empty, there is an issue with memory. Only for debugging purposes. + ''' + for b in self.branches: + print prefix, b.GetName(), repr(b.GetAddress()) + def cloneWithoutCompile(self, externalTree = None): ''' make a deep copy of self to e.g. avoid re-compilation of class in a loop. Reset TTree as to not create a memory leak. @@ -54,6 +60,7 @@ def cloneWithoutCompile(self, externalTree = None): res.tree = ROOT.TTree( treeName, treeName ) res.makeBranches() + #self.debugBranchAddresses(prefix = "cloneWithoutCompile") return res @@ -81,7 +88,7 @@ def makeBranches(self): vectorCount+=1 else: raise ValueError( "Don't know what variable %r is." % s ) - + #self.debugBranchAddresses(prefix = "makeBranches") logger.debug( "TreeMaker created %i new scalars and %i new vectors.", scalerCount, vectorCount ) def clear(self): @@ -89,6 +96,7 @@ def clear(self): def fill(self): # Write to TTree + #self.debugBranchAddresses( prefix = "Filling") if self.treeIsExternal: for b in self.branches: b.Fill() diff --git a/core/python/TreeReader.py b/core/python/TreeReader.py index be4822b..1030a7f 100644 --- a/core/python/TreeReader.py +++ b/core/python/TreeReader.py @@ -45,8 +45,8 @@ def __init__(self, sample, variables=[], sequence = [], selectionString = None, # Whether all branches are to be read or whether that information should come from the variables self.allBranchesActive = allBranchesActive - # Read branch information from the chain - self.readLeafInfo() + ## Read branch information from the chain -> Will be useful when using auto-type + #self.readLeafInfo() # 'variables' are read from the chain super(TreeReader, self).__init__( variables = list(set(variables)) ) @@ -59,18 +59,19 @@ def __init__(self, sample, variables=[], sequence = [], selectionString = None, # set the addresses of the branches corresponding to 'variables' self.setAddresses() - + # make eList from cutString # Turn on everything for flexibility with the selectionString logger.debug("Initializing TreeReader for sample %s", self.sample.name) self.activateAllBranches() - self.eList = self.sample.getEventList(selectionString = self.selectionString) if self.selectionString is not None else None + self._eList = self.sample.getEventList(selectionString = self.selectionString) if self.selectionString is not None else None self.activateBranches() - self.nEvents = self.eList.GetN() if self.eList else self.sample.chain.GetEntries() + self.nEvents = self._eList.GetN() if self._eList else self.sample.chain.GetEntries() logger.debug("Found %i events in %s", self.nEvents, self.sample.name) # default event range of the reader self.eventRange = (0, self.nEvents) + def setAddresses(self): ''' Set all the branch addresses to the members in the class instance @@ -84,26 +85,25 @@ def setAddresses(self): self.sample.chain.SetBranchAddress(comp.name, ROOT.AddressOf(self.event, comp.name )) else: raise ValueError( "Don't know what variable %r is." % s ) - def cloneTree(self, branchList = [], newTreename = None, rootfile = None): '''Clone tree after preselection and event range ''' selectionString = self.selectionString if self.selectionString is not None else "1" - if self.eList: + if self._eList: # If there is an eList, first restrict it to the event range, then clone + import uuid + name = str(uuid.uuid4()) list_to_copy = ROOT.TEventList("tmp","tmp") for i_ev in xrange(*self.eventRange): - list_to_copy.Enter(self.eList.GetEntry(i_ev)) + list_to_copy.Enter(self._eList.GetEntry(i_ev)) + + self.sample.chain.GetEntry(list_to_copy.GetEntry(0)) #This is needed to keep branch addresses when running over a few events and >=1 file # activate branches that we want to copy, disable the ones we only need for reading self.activateBranches( turnOnReadBranches = False, branchList = branchList ) - # preserving current event list - tmpEventList = self.sample.chain.GetEventList() - tmpEventList = 0 if not self.sample.chain.GetEventList() else tmpEventList # Copy only the selected events - self.sample.chain.SetEventList( list_to_copy ) # Create the new tree in a file (if there is one) tmp_directory = ROOT.gDirectory @@ -113,42 +113,19 @@ def cloneTree(self, branchList = [], newTreename = None, rootfile = None): # Copying tree - # 1. causes trouble with few events and few files. - # res = self.sample.chain.CopyTree( "(1)", "" ) - - # 1.5: this seems to give misaligned l1_pt and LepGood_pt??? - # tree = self.sample.chain.GetTree() - # tree.SetEventList( list_to_copy ) - # res = tree.CopyTree( "(1)", "" ) - - # 2. Doesn't take into account event list - #res = self.sample.chain.CloneTree( 0 ) - #res.CopyEntries( self.sample.chain ) - - # 3. Loop (no problems observed) logger.debug("Copying %i events in a loop.", list_to_copy.GetN()) - #tree = self.sample.chain.GetTree() - res = self.sample.chain.GetTree().CloneTree( 0 ) + res = self.sample.chain.GetTree().CloneTree( 0 ) for i_event in xrange(list_to_copy.GetN()): self.sample.chain.GetEntry( list_to_copy.GetEntry(i_event) ) res.Fill() - # Needed? - res.Write() - ## 4. Doesn't take into account event list - #tree = self.sample.chain.GetTree() - #tree.SetEventList( tmpEventList ) - #res = tree.CloneTree( 0 ) - #res.CopyEntries( tree ) + res.Write() logger.debug("Number of events: list_to_copy %i res.GetEntries() %i", list_to_copy.GetN(), res.GetEntries()) # Change back to previous gDirectory tmp_directory.cd() - # restoring event list - self.sample.chain.SetEventList( tmpEventList ) - # activate what we read, don't activate the ones we just copied self.activateBranches( turnOnReadBranches = True, branchList = [] ) @@ -197,28 +174,28 @@ def activateAllBranches(self): ''' self.sample.chain.SetBranchStatus("*", 1) - def readLeafInfo(self): - ''' Read information on the leaves from the chain and store in dict - FIXME: Pretty sure this doesn't yet work for fixed sized vectors - ''' - leafInfo = [] - for s in self.sample.chain.GetListOfLeaves(): - leaf = {'type':shortTypeDict[s.GetTypeName()], 'name':s.GetName()} - countval = array.array('i',[-9999]) - pointer = s.GetLeafCounter(countval) - leaf['dim'] = 'scalar' - if pointer: - # Vector - # GetLeafCounter returns countval==1 if a counter leaf was found and pointer points to that leaf - # https://root.cern.ch/doc/master/classTLeaf.html#a062bd89a11fd1f922c096e66d2601ab6 - if countval[0]==1: - leaf['counterInt'] = pointer.GetName() - leaf['dim'] = 'vector' - else: - # For fixed size arrays, the pointer is zero and the countval is the size of the array - leaf['nMax'] = countval[0] - leafInfo.append(leaf) - return leafInfo +# def readLeafInfo(self): +# ''' Read information on the leaves from the chain and store in dict +# FIXME: Pretty sure this doesn't yet work for fixed sized vectors +# ''' +# leafInfo = [] +# for s in self.sample.chain.GetListOfLeaves(): +# leaf = {'type':shortTypeDict[s.GetTypeName()], 'name':s.GetName()} +# countval = array.array('i',[-9999]) +# pointer = s.GetLeafCounter(countval) +# leaf['dim'] = 'scalar' +# if pointer: +# # Vector +# # GetLeafCounter returns countval==1 if a counter leaf was found and pointer points to that leaf +# # https://root.cern.ch/doc/master/classTLeaf.html#a062bd89a11fd1f922c096e66d2601ab6 +# if countval[0]==1: +# leaf['counterInt'] = pointer.GetName() +# leaf['dim'] = 'vector' +# else: +# # For fixed size arrays, the pointer is zero and the countval is the size of the array +# leaf['nMax'] = countval[0] +# leafInfo.append(leaf) +# return leafInfo def getEventRanges(self, maxFileSizeMB = None, maxNEvents = None, nJobs = None, minJobs = None): '''For convinience: Define splitting of sample according to various criteria @@ -252,14 +229,14 @@ def setEventRange( self, evtRange ): self.eventRange = ( max(0, evtRange[0]), min( self.nEvents, evtRange[1]) ) logger.debug( "[setEventRange] Set eventRange %r (was: %r) for reader of sample %s", self.eventRange, old_eventRange, self.sample.name ) - def setEventList( self, evtList ): - ''' Specify an event list that the reader will run over. - ''' - self.sample.chain.SetEventList( evtList ) - self.eList = evtList - self.nEvents = self.eList.GetN() - self.eventRange = (0, self.nEvents) - logger.debug( "[setEventList] Set eventRange %r for reader of sample %s", self.eventRange, self.sample.name ) + #def setEventList( self, evtList ): + # ''' Specify an event list that the reader will run over. + # ''' + # self.sample.chain.SetEventList( evtList ) + # self.eList = evtList + # self.nEvents = self.eList.GetN() + # self.eventRange = (0, self.nEvents) + # logger.debug( "[setEventList] Set eventRange %r for reader of sample %s", self.eventRange, self.sample.name ) def reduceEventRange( self, reduction_factor ): ''' Reduce event range by a given factor. From f8727cc17fb85eea20fc202f9f91ba14bfe206a6 Mon Sep 17 00:00:00 2001 From: Robert Schoefbeck Date: Wed, 8 Mar 2017 10:34:13 +0100 Subject: [PATCH 04/16] fix bug when running over a small number of events. More care with event lists --- core/python/TreeReader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/python/TreeReader.py b/core/python/TreeReader.py index 1030a7f..07cc1a4 100644 --- a/core/python/TreeReader.py +++ b/core/python/TreeReader.py @@ -276,7 +276,7 @@ def _execute(self): # get entry errorLevel = ROOT.gErrorIgnoreLevel ROOT.gErrorIgnoreLevel = 3000 - self.sample.chain.GetEntry ( self.eList.GetEntry( self.position ) ) if self.eList else self.sample.chain.GetEntry( self.position ) + self.sample.chain.GetEntry ( self._eList.GetEntry( self.position ) ) if self._eList else self.sample.chain.GetEntry( self.position ) ROOT.gErrorIgnoreLevel = errorLevel # sequence From 59a5cd40339c5f07d3f26cd0d665326426785aea Mon Sep 17 00:00:00 2001 From: UAEDF-tomc Date: Wed, 8 Mar 2017 16:17:10 +0100 Subject: [PATCH 05/16] could have more columns in the legend (backwards compatible) --- plot/python/plotting.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/plot/python/plotting.py b/plot/python/plotting.py index ef9885d..d9b30f5 100644 --- a/plot/python/plotting.py +++ b/plot/python/plotting.py @@ -341,6 +341,12 @@ def draw(plot, \ max_ = max( l[0].GetMaximum() for l in histos ) min_ = min( l[0].GetMinimum() for l in histos ) + # If legend is in the form (tuple, int) then the number of columns is provided + legendColumns = 1 + if len(legend) == 2: + legendColumns = legend[1] + legend = legend[0] + #Calculate legend coordinates in gPad coordinates if legend is not None: if legend=="auto": @@ -424,6 +430,7 @@ def draw(plot, \ # Make the legend if legend is not None: legend_ = ROOT.TLegend(*legendCoordinates) + legend_.SetNColumns(legendColumns) legend_.SetFillStyle(0) # legend_.SetFillColor(0) legend_.SetShadowColor(ROOT.kWhite) From 687e61f25cb5ffe3eb71e04ee5c892f105accdfe Mon Sep 17 00:00:00 2001 From: Robert Schoefbeck Date: Thu, 9 Mar 2017 06:47:15 +0100 Subject: [PATCH 06/16] docstring for legend --- plot/python/plotting.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/plot/python/plotting.py b/plot/python/plotting.py index d9b30f5..cdb56eb 100644 --- a/plot/python/plotting.py +++ b/plot/python/plotting.py @@ -222,9 +222,9 @@ def draw(plot, \ extensions: ["pdf", "png", "root"] (default) logX: True/False (default), logY: True(default)/False ratio: 'auto'(default) corresponds to {'num':1, 'den':0, 'logY':False, 'style':None, 'texY': 'Data / MC', 'yRange': (0.5, 1.5), 'drawObjects': []} - scaling: {} (default). Scaling the i-th stach to the j-th is done by scaling = {i:j} with i,j integers + scaling: {} (default). Scaling the i-th stack to the j-th is done by scaling = {i:j} with i,j integers sorting: True/False(default) Whether or not to sort the components of a stack wrt Integral - legend: "auto" (default) or [x_low, y_low, x_high, y_high] or None + legend: "auto" (default) or [x_low, y_low, x_high, y_high] or None. ([], n) divides the legend into n columns. drawObjects = [] Additional ROOT objects that are called by .Draw() widths = {} (default) to update the widths. Values are {'y_width':500, 'x_width':500, 'y_ratio_width':200} canvasModifications = [] could be used to pass on lambdas to modify the canvas @@ -262,7 +262,6 @@ def draw(plot, \ for p in s: Plot.addOverFlowBin1D( p, plot.addOverFlowBin ) - for i, l in enumerate(histos): # recall the sample for use in the legend From f516d8a24627c9644a957fe6ad6625db4ee078b7 Mon Sep 17 00:00:00 2001 From: Robert Schoefbeck Date: Thu, 9 Mar 2017 09:32:15 +0100 Subject: [PATCH 07/16] Sample.split( n ) functionality --- core/python/Sample.py | 22 ++++++++++++++++++++++ core/python/TreeReader.py | 5 ----- core/python/helpers.py | 9 ++++++++- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/core/python/Sample.py b/core/python/Sample.py index 59a1e93..33579c9 100644 --- a/core/python/Sample.py +++ b/core/python/Sample.py @@ -404,6 +404,28 @@ def fromCMGCrabDirectory(cls, name, baseDirectory, treeFilename = 'tree.root', t selectionString = selectionString, weightString = weightString, isData = isData, color = color, texName = texName ) + def split( self, n, clear = True): + ''' Split sample into n sub-samples + ''' + if not n>=1: + raise ValueError( "Can not split into: '%r'" % n ) + + chunks = helpers.partition( self.files, min(n , len(self.files) ) ) + + if clear: self.clear() # Kill yourself. + + return [ Sample( + name = self.name+"_%i" % n_sample, + treeName = self.treeName, + files = chunks[n_sample], + normalization = self.normalization, + selectionString = self.selectionString, + weightString = self.weightString, + isData = self.isData, + color = self.color, + texName = self.texName ) for n_sample in xrange(len(chunks)) ] + + # Handle loading of chain -> load it when first used @property def chain(self): diff --git a/core/python/TreeReader.py b/core/python/TreeReader.py index 07cc1a4..57a6c36 100644 --- a/core/python/TreeReader.py +++ b/core/python/TreeReader.py @@ -200,11 +200,6 @@ def activateAllBranches(self): def getEventRanges(self, maxFileSizeMB = None, maxNEvents = None, nJobs = None, minJobs = None): '''For convinience: Define splitting of sample according to various criteria ''' - def chunks(l, n): - """Yield successive n-sized chunks from l.""" - for i in xrange(0, len(l), n): - yield (i, i+n) - if maxFileSizeMB is not None: nSplit = sum( os.path.getsize(f) for f in self.sample.files ) / ( 1024**2*maxFileSizeMB ) elif maxNEvents is not None: diff --git a/core/python/helpers.py b/core/python/helpers.py index 5e61a0e..6c358e6 100644 --- a/core/python/helpers.py +++ b/core/python/helpers.py @@ -5,11 +5,19 @@ import logging logger = logging.getLogger(__name__) +# Error handling class EmptySampleError(Exception): '''Accessing a sample without ROOT files. ''' pass +# List helper +def partition(lst, n): + ''' Partition list into chunks of approximately equal size''' + # http://stackoverflow.com/questions/2659900/python-slicing-a-list-into-n-nearly-equal-length-partitions + n_division = len(lst) / float(n) + return [ lst[int(round(n_division * i)): int(round(n_division * (i + 1)))] for i in xrange(n) ] + # Translation of short types to ROOT C types cStringTypeDict = { 'b': 'UChar_t', @@ -49,7 +57,6 @@ def decorate(func): return func return decorate - def checkRootFile(f, checkForObjects=[] ): ''' Checks whether a root file exists, was not recoverd or otherwise broken and contains the objects in 'checkForObjects' From b922f4e9500823481de0080ac9c16652648acf75 Mon Sep 17 00:00:00 2001 From: UAEDF-tomc Date: Fri, 10 Mar 2017 11:19:16 +0100 Subject: [PATCH 08/16] hist modifications added --- plot/python/plotting.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plot/python/plotting.py b/plot/python/plotting.py index d9b30f5..cfbf392 100644 --- a/plot/python/plotting.py +++ b/plot/python/plotting.py @@ -216,6 +216,7 @@ def draw(plot, \ drawObjects = [], widths = {}, canvasModifications = [], + histModifications = [], copyIndexPHP = False ): ''' yRange: 'auto' (default) or [low, high] where low/high can be 'auto' @@ -423,6 +424,7 @@ def draw(plot, \ else: h.GetYaxis().SetTitleOffset( 1.6 ) + for modification in histModifications: modification(h) h.Draw(drawOption+same) same = "same" From 6e3732bc3e2facf1c58fadfa5849fe65b5cff9d1 Mon Sep 17 00:00:00 2001 From: Robert Schoefbeck Date: Fri, 10 Mar 2017 14:29:34 +0100 Subject: [PATCH 09/16] small fix in one of the examples --- examples/example_plot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example_plot.py b/examples/example_plot.py index 191fb86..8804f25 100644 --- a/examples/example_plot.py +++ b/examples/example_plot.py @@ -51,7 +51,7 @@ stack = Stack( [ s0, s1], [ s2 ] ) # Let's use a trivial weight. All functions will -plot_weight = lambda data:1 +plot_weight = lambda event, sample : 1 # Two selection strings selectionString = "nJet>0" From 6ee7270116a788571708ade106b9c7392086eecd Mon Sep 17 00:00:00 2001 From: UAEDF-tomc Date: Fri, 10 Mar 2017 14:36:01 +0100 Subject: [PATCH 10/16] making sure error bars are drawn in the ratio plot even if the central value is off scale --- plot/python/plotting.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/plot/python/plotting.py b/plot/python/plotting.py index b589afd..b586650 100644 --- a/plot/python/plotting.py +++ b/plot/python/plotting.py @@ -424,6 +424,7 @@ def draw(plot, \ h.GetYaxis().SetTitleOffset( 1.6 ) for modification in histModifications: modification(h) + if drawOption=="e1": dataHist = h h.Draw(drawOption+same) same = "same" @@ -497,6 +498,14 @@ def draw(plot, \ drawOption = h_ratio.drawOption if hasattr(h_ratio, "drawOption") else "hist" h_ratio.Draw(drawOption) + if drawOption == "e1": # hacking to show error bars within panel when central value is off scale + graph = ROOT.TGraphAsymmErrors(dataHist) # cloning from datahist in order to get layout + for bin in range(1, h_ratio.GetNbinsX()+1): + val = h_ratio.GetBinContent(bin) + err = h_ratio.GetBinError(bin) + graph.SetPoint(bin, bin-0.5, val) + graph.SetPointError(bin, 0, 0, err, err) + graph.Draw("P0 same") bottomPad.SetLogx(logX) bottomPad.SetLogy(ratio['logY']) From 5885aa5d8c7e8f119ee00100f831a90100fccc97 Mon Sep 17 00:00:00 2001 From: UAEDF-tomc Date: Mon, 13 Mar 2017 15:04:37 +0100 Subject: [PATCH 11/16] poissonian error bars in ratio plot --- plot/python/plotting.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/plot/python/plotting.py b/plot/python/plotting.py index b586650..0ae580f 100644 --- a/plot/python/plotting.py +++ b/plot/python/plotting.py @@ -460,7 +460,6 @@ def draw(plot, \ if ratio is not None: bottomPad.cd() num = histos[ratio['num']][0] - h_ratio = helpers.clone( num ) # For a ratio of profiles, use projection (preserve attributes) @@ -497,15 +496,19 @@ def draw(plot, \ h_ratio.GetYaxis().SetNdivisions(505) drawOption = h_ratio.drawOption if hasattr(h_ratio, "drawOption") else "hist" - h_ratio.Draw(drawOption) if drawOption == "e1": # hacking to show error bars within panel when central value is off scale graph = ROOT.TGraphAsymmErrors(dataHist) # cloning from datahist in order to get layout - for bin in range(1, h_ratio.GetNbinsX()+1): - val = h_ratio.GetBinContent(bin) - err = h_ratio.GetBinError(bin) + for bin in range(1, h_ratio.GetNbinsX()+1): # do not show error bars on hist + h_ratio.SetBinError(bin, 0.0001) + val = h_ratio.GetBinContent(bin) + errUp = num.GetBinErrorUp(bin)/histos[ratio['den']][0].GetBinContent(bin) if val > 0 else 0 + errDown = num.GetBinErrorLow(bin)/histos[ratio['den']][0].GetBinContent(bin) if val > 0 else 0 graph.SetPoint(bin, bin-0.5, val) - graph.SetPointError(bin, 0, 0, err, err) + graph.SetPointError(bin, 0, 0, errDown, errUp) + h_ratio.Draw("e0") graph.Draw("P0 same") + else: + h_ratio.Draw(drawOption) bottomPad.SetLogx(logX) bottomPad.SetLogy(ratio['logY']) From 51a979c693c7e8074d00a2e8905ca312e9548a22 Mon Sep 17 00:00:00 2001 From: Robert Schoefbeck Date: Thu, 6 Apr 2017 11:24:39 +0200 Subject: [PATCH 12/16] ValueError log msg for Sample.combine --- core/python/Sample.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/python/Sample.py b/core/python/Sample.py index 33579c9..be871fa 100644 --- a/core/python/Sample.py +++ b/core/python/Sample.py @@ -27,7 +27,7 @@ def new_name(): def check_equal_(vals): if not len(set(vals)) == 1: - raise ValueError( "These values should be identical but are not: %r"%vals ) + raise ValueError( "Sample combine check failed on: %r"%vals ) else: return vals[0] From bc52f7b3199ebc0451ce4e548783285981685521 Mon Sep 17 00:00:00 2001 From: UAEDF-tomc Date: Thu, 6 Apr 2017 11:43:24 +0200 Subject: [PATCH 13/16] attempt to solve problem with duplicate points in ratio plot --- plot/python/plotting.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plot/python/plotting.py b/plot/python/plotting.py index 0ae580f..9b078ca 100644 --- a/plot/python/plotting.py +++ b/plot/python/plotting.py @@ -498,12 +498,14 @@ def draw(plot, \ drawOption = h_ratio.drawOption if hasattr(h_ratio, "drawOption") else "hist" if drawOption == "e1": # hacking to show error bars within panel when central value is off scale graph = ROOT.TGraphAsymmErrors(dataHist) # cloning from datahist in order to get layout + graph.Set(0) for bin in range(1, h_ratio.GetNbinsX()+1): # do not show error bars on hist h_ratio.SetBinError(bin, 0.0001) + center = h_ratio.GetBinCenter(bin) val = h_ratio.GetBinContent(bin) errUp = num.GetBinErrorUp(bin)/histos[ratio['den']][0].GetBinContent(bin) if val > 0 else 0 errDown = num.GetBinErrorLow(bin)/histos[ratio['den']][0].GetBinContent(bin) if val > 0 else 0 - graph.SetPoint(bin, bin-0.5, val) + graph.SetPoint(bin, center, val) graph.SetPointError(bin, 0, 0, errDown, errUp) h_ratio.Draw("e0") graph.Draw("P0 same") From b9feeca76dc864cb6f911b3b9989cff30e56e7df Mon Sep 17 00:00:00 2001 From: Robert Schoefbeck Date: Thu, 1 Jun 2017 21:22:10 +0200 Subject: [PATCH 14/16] resolve rare race condition when plotting --- plot/python/plotting.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plot/python/plotting.py b/plot/python/plotting.py index cdb56eb..dec939d 100644 --- a/plot/python/plotting.py +++ b/plot/python/plotting.py @@ -512,7 +512,10 @@ def draw(plot, \ logger.debug( "ratio['drawObjects'] has something I can't Draw(): %r", o) if not os.path.exists(plot_directory): - os.makedirs(plot_directory) + try: + os.makedirs(plot_directory) + except OSError: # Resolve rare race condition + pass if copyIndexPHP: plot_helpers.copyIndexPHP( plot_directory ) From 10c560c85865e1cfea9a1cb32c766c6a8e547887 Mon Sep 17 00:00:00 2001 From: Robert Schoefbeck Date: Tue, 13 Jun 2017 09:01:46 +0200 Subject: [PATCH 15/16] FWLiteSample fromDAS --- fwlite/python/FWLiteSample.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/fwlite/python/FWLiteSample.py b/fwlite/python/FWLiteSample.py index b42e08f..a4f8189 100644 --- a/fwlite/python/FWLiteSample.py +++ b/fwlite/python/FWLiteSample.py @@ -85,11 +85,10 @@ def fromDirectory(cls, name, directory, color = 0, texName = None, maxN = None): maxN = maxN if maxN is not None and maxN>0 else None files = files[:maxN] - sample = cls(name = name, files = files, color=color, texName = texName) - return sample + return cls(name = name, files = files, color=color, texName = texName) @classmethod - def fromDAS(cls, name, dataset, instance = 'global', prefix='root://cms-xrd-global.cern.ch/', maxN = None): + def fromDAS(cls, name, dataset, instance = 'global', prefix='root://cms-xrd-global.cern.ch/', texName = None, maxN = None): ''' Make sample from DAS. ''' # https://github.com/CERN-PH-CMG/cmg-cmssw/blob/0f1d3bf62e7ec91c2e249af1555644b7f414ab50/CMGTools/Production/python/dataset.py#L437 @@ -119,8 +118,24 @@ def _dasPopen(dbs): line = line.rstrip() files.append(prefix+line) - return cls(name, files=files) + return cls(name, files=files, texName = texName) + + @classmethod + def combine(cls, name, samples, texName = None, maxN = None, color = 0): + '''Make new sample from a list of samples. + ''' + if not (type(samples) in [type([]), type(())]) or len(samples)<1: + raise ValueError( "Need non-empty list of samples. Got %r"% samples) + + files = sum([s.files for s in samples], []) + maxN = maxN if maxN is not None and maxN>0 else None + files = files[:maxN] + return cls(name = name, \ + files = files, + color = color, + texName = texName + ) def fwliteReader(self, **kwargs): ''' Return a FWLiteReader class for the sample From a8bca9989055934644c96a3f6bf072c08a17a3c1 Mon Sep 17 00:00:00 2001 From: Robert Schoefbeck Date: Tue, 13 Jun 2017 09:01:59 +0200 Subject: [PATCH 16/16] max_events in plotting.fill --- plot/python/plotting.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/plot/python/plotting.py b/plot/python/plotting.py index 9b078ca..c7b3fdc 100644 --- a/plot/python/plotting.py +++ b/plot/python/plotting.py @@ -32,7 +32,7 @@ def constrain(x, interval=[0,1]): 'xUpperEdge':constrain( (legend_coordinates[2] - pad.GetLeftMargin())/(1.-pad.GetLeftMargin()-pad.GetRightMargin()), interval = [0, 1] ) } -def fill(plots, read_variables = [], sequence=[] ): +def fill(plots, read_variables = [], sequence=[], max_events = -1 ): '''Create histos and fill all plots ''' @@ -117,6 +117,7 @@ def fill(plots, read_variables = [], sequence=[] ): plot.store_fillers = plot.fillers r.start() + counter = 0 while r.run(): for plot in plots_for_sample: for index in plot.sample_indices: @@ -134,6 +135,12 @@ def fill(plots, read_variables = [], sequence=[] ): TH_fill_args.append( weight*sample_scale_factor ) plot.histos[index[0]][index[1]].Fill( *TH_fill_args ) + if max_events > 0: + counter += 1 + if counter > max_events: + logger.debug( "Stop filling histograms because counter is %i and max_events is %i", counter, max_events ) + break + # Clean up for plot in plots_for_sample: del plot.sample_indices