Enums

[mrocklin]

This fails

import enum
class MyEnum(enum.Enum):
 SPAM = 'SPAM'

import cloudpickle
cloudpickle.dumps(MyEnum.SPAM)

---------------------------------------------------------------------------
RecursionError Traceback (most recent call last)
/home/mrocklin/Software/anaconda/lib/python3.6/site-packages/cloudpickle/cloudpickle.py in dump(self, obj)
 145 try:
--> 146 return Pickler.dump(self, obj)
 147 except RuntimeError as e:

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in dump(self, obj)
 408 self.framer.start_framing()
--> 409 self.save(obj)
 410 self.write(STOP)

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
 520 # Save the reduce() output and finally memoize the object
--> 521 self.save_reduce(obj=obj, *rv)
 522 

/home/mrocklin/Software/anaconda/lib/python3.6/site-packages/cloudpickle/cloudpickle.py in save_reduce(self, func, args, state, listitems, dictitems, obj)
 584 else:
--> 585 save(func)
 586 save(args)

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
 489 if issc:
--> 490 self.save_global(obj)
 491 return

/home/mrocklin/Software/anaconda/lib/python3.6/site-packages/cloudpickle/cloudpickle.py in save_global(self, obj, name, pack)
 424 
--> 425 self.save_reduce(typ, (obj.__name__, obj.__bases__, d), obj=obj)
 426 else:

/home/mrocklin/Software/anaconda/lib/python3.6/site-packages/cloudpickle/cloudpickle.py in save_reduce(self, func, args, state, listitems, dictitems, obj)
 585 save(func)
--> 586 save(args)
 587 write(pickle.REDUCE)

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
 475 if f is not None:
--> 476 f(self, obj) # Call unbound method with explicit self
 477 return

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in save_tuple(self, obj)
 735 for element in obj:
--> 736 save(element)
 737 # Subtle. Same as in the big comment below.

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
 475 if f is not None:
--> 476 f(self, obj) # Call unbound method with explicit self
 477 return

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in save_dict(self, obj)
 820 self.memoize(obj)
--> 821 self._batch_setitems(obj.items())
 822 

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in _batch_setitems(self, items)
 846 save(k)
--> 847 save(v)
 848 write(SETITEMS)

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
 520 # Save the reduce() output and finally memoize the object
--> 521 self.save_reduce(obj=obj, *rv)
 522 

/home/mrocklin/Software/anaconda/lib/python3.6/site-packages/cloudpickle/cloudpickle.py in save_reduce(self, func, args, state, listitems, dictitems, obj)
 600 if dictitems is not None:
--> 601 self._batch_setitems(dictitems)
 602 

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in _batch_setitems(self, items)
 851 save(k)
--> 852 save(v)
 853 write(SETITEM)

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
 520 # Save the reduce() output and finally memoize the object
--> 521 self.save_reduce(obj=obj, *rv)
 522 

/home/mrocklin/Software/anaconda/lib/python3.6/site-packages/cloudpickle/cloudpickle.py in save_reduce(self, func, args, state, listitems, dictitems, obj)
 584 else:
--> 585 save(func)
 586 save(args)

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
 489 if issc:
--> 490 self.save_global(obj)
 491 return

/home/mrocklin/Software/anaconda/lib/python3.6/site-packages/cloudpickle/cloudpickle.py in save_global(self, obj, name, pack)
 424 
--> 425 self.save_reduce(typ, (obj.__name__, obj.__bases__, d), obj=obj)
 426 else:

/home/mrocklin/Software/anaconda/lib/python3.6/site-packages/cloudpickle/cloudpickle.py in save_reduce(self, func, args, state, listitems, dictitems, obj)
 585 save(func)
--> 586 save(args)
 587 write(pickle.REDUCE)

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
 475 if f is not None:
--> 476 f(self, obj) # Call unbound method with explicit self
 477 return

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in save_tuple(self, obj)
 735 for element in obj:
--> 736 save(element)
 737 # Subtle. Same as in the big comment below.

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
 475 if f is not None:
--> 476 f(self, obj) # Call unbound method with explicit self
 477 return

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in save_dict(self, obj)
 820 self.memoize(obj)
--> 821 self._batch_setitems(obj.items())
 822 

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in _batch_setitems(self, items)
 846 save(k)
--> 847 save(v)
 848 write(SETITEMS)

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
 475 if f is not None:
--> 476 f(self, obj) # Call unbound method with explicit self
 477 return

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in save_dict(self, obj)
 820 self.memoize(obj)
--> 821 self._batch_setitems(obj.items())
 822 

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in _batch_setitems(self, items)
 851 save(k)
--> 852 save(v)
 853 write(SETITEM)

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
 520 # Save the reduce() output and finally memoize the object
--> 521 self.save_reduce(obj=obj, *rv)
 522 

/home/mrocklin/Software/anaconda/lib/python3.6/site-packages/cloudpickle/cloudpickle.py in save_reduce(self, func, args, state, listitems, dictitems, obj)
 584 else:
--> 585 save(func)
 586 save(args)

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
 489 if issc:
--> 490 self.save_global(obj)
 491 return

/home/mrocklin/Software/anaconda/lib/python3.6/site-packages/cloudpickle/cloudpickle.py in save_global(self, obj, name, pack)
 424 
--> 425 self.save_reduce(typ, (obj.__name__, obj.__bases__, d), obj=obj)
 426 else:

/home/mrocklin/Software/anaconda/lib/python3.6/site-packages/cloudpickle/cloudpickle.py in save_reduce(self, func, args, state, listitems, dictitems, obj)
 585 save(func)
--> 586 save(args)
 587 write(pickle.REDUCE)

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
 475 if f is not None:
--> 476 f(self, obj) # Call unbound method with explicit self
 477 return

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in save_tuple(self, obj)
 735 for element in obj:
--> 736 save(element)
 737 # Subtle. Same as in the big comment below.

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
 475 if f is not None:
--> 476 f(self, obj) # Call unbound method with explicit self
 477 return

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in save_dict(self, obj)
 820 self.memoize(obj)
--> 821 self._batch_setitems(obj.items())
 822 

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in _batch_setitems(self, items)
 846 save(k)
--> 847 save(v)
 848 write(SETITEMS)

... last 10 frames repeated, from the frame below ...

/home/mrocklin/Software/anaconda/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
 520 # Save the reduce() output and finally memoize the object
--> 521 self.save_reduce(obj=obj, *rv)
 522 

RecursionError: maximum recursion depth exceeded

During handling of the above exception, another exception occurred:

PicklingError Traceback (most recent call last)
<ipython-input-3-d64a2267ce31> in <module>()
----> 1 cloudpickle.dumps(MyEnum.SPAM)

/home/mrocklin/Software/anaconda/lib/python3.6/site-packages/cloudpickle/cloudpickle.py in dumps(obj, protocol)
 704 
 705 cp = CloudPickler(file,protocol)
--> 706 cp.dump(obj)
 707 
 708 return file.getvalue()

/home/mrocklin/Software/anaconda/lib/python3.6/site-packages/cloudpickle/cloudpickle.py in dump(self, obj)
 148 if 'recursion' in e.args[0]:
 149 msg = """Could not pickle object as excessively deep recursion required."""
--> 150 raise pickle.PicklingError(msg)
 151 
 152 def save_memoryview(self, obj):

PicklingError: Could not pickle object as excessively deep recursion required.

Originally reported here: dask/distributed#1178 by @AndrewPashkin

[llllllllll]

I believe the root of this issue is that the class dict holds instances of the type it.
A more simple reproduction case is:

In [1]: class C:
 ...: pass
 ...: 

In [2]: C.c = C()

In [3]: cp.dumps(C)

[AndreiPashkin]

That's correct. Standard pickle can pickle that cases:

In [64]: class C():
 ...: pass
 ...: 

In [65]: C.c = C()

In [66]: import pickle

In [67]: pickle.dumps(C)
Out[67]: b'\x80\x03c__main__\nC\nq\x00.

In [77]: assert hasattr(pickle.loads(pickle.dumps(C)), 'c')

[ssanderson]

@mrocklin @AndrewPashkin as of #102 trying to serialize an enum no longer triggers an infinite recursion, and it works for enums defined as non-__main__ globals (that may have already been the case before). For dynamic enums, we now crash with an error in the Enum metaclass while deserializing. Assuming we want cloudpickle to support dynamic Enums, the right fix is probably to just add special case dispatches for serializing Enum and EnumMeta, since they're standard library types.

One worry I'd have about supporting dynamic enums is that it's idiomatic to use is to compare Enum values, which could cause unexpected results if users compare values produced by separate calls to loads. Do you have a sense of whether that would be significant footgun in practical usage of cloudpickle?

[mrocklin]

I personally have no practical experience with Enum types. I was just pushing a Dask error upstream here.

[AndreiPashkin]

@ssanderson

it's idiomatic to use is to compare Enum values

Indeed:
https://docs.python.org/3/library/enum.html#comparisons

Do you have a sense of whether that would be significant footgun in practical usage of cloudpickle?

Would is don't work only in case of dynamic Enums?

[ssanderson]

@AndrewPashkin is would work sometimes. a is b means "a occupies the same memory as b", which means that is comparisons wouldn't work between enum members that were unpickled by different calls to load. For example, if you have a pickled object containing values from an Enum and you unpickle that object twice, you'll get two different copies of the Enum, which means is comparisons between the two copies of the Enum won't work as expected.

[chebee7i]

Is this expected soon?

[jakirkham]

Have retested the OP code with cloudpickle version 0.5.2 on Python 3 and it runs correctly without exception.

That said, it appears the round trip fails. Perhaps related to issue ( #117 ).

Details

In [1]: import enum
 ...: class MyEnum(enum.Enum):
 ...: SPAM = 'SPAM'
 ...: 
 ...: import cloudpickle
 ...: cloudpickle.loads(cloudpickle.dumps(MyEnum.SPAM))
 ...: 
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-1-3cc75bf49a58> in <module>()
 4 
 5 import cloudpickle
----> 6 cloudpickle.loads(cloudpickle.dumps(MyEnum.SPAM))

/zopt/conda2/envs/nanshenv3/lib/python3.6/enum.py in __new__(metacls, cls, bases, classdict)
 133 # save enum items into separate mapping so they don't get baked into
 134 # the new class
--> 135 enum_members = {k: classdict[k] for k in classdict._member_names}
 136 for name in classdict._member_names:
 137 del classdict[name]

AttributeError: 'dict' object has no attribute '_member_names'

[ogrisel]

For reference, I started a new similar PR in #246 but as discussed in the comments, I don't think the current cloudpickle behavior is fine. We would need an explicit scoping mechanism to make it possible to load 2 pickle strings that hold instances of the same original dynamically class / types to share the reconstructed type definitions while also making it possible to change redefine classes with the same name in the original __main__ module and have those code changes reflected in subsequent dumps / loads on worker nodes.

[pierreglaser]

Now that #246 is merged, we can close this.

[jakirkham]

Confirmed as fixed.

In [1]: import enum 
 ...: class MyEnum(enum.Enum): 
 ...: SPAM = 'SPAM' 
 ...: 
 ...: import cloudpickle 
 ...: cloudpickle.dumps(MyEnum.SPAM) 
Out[1]: b'\x80\x04\x95\xe9\x00\x00\x00\x00\x00\x00\x00\x8c\x17cloudpickle.cloudpickle\x94\x8c\x19_rehydrate_skeleton_class\x94\x93\x94(h\x00\x8c\x13_make_skeleton_enum\x94\x93\x94(\x8c\x04enum\x94\x8c\x04Enum\x94\x93\x94\x85\x94\x8c\x06MyEnum\x94h\t}\x94\x8c\x04SPAM\x94h\x0bs\x8c\x08__main__\x94\x8c a3c0e0e72031415e9f99a4c26e525d65\x94Nt\x94R\x94}\x94(\x8c\n__module__\x94h\x0c\x8c\x07__new__\x94h\x05\x8c\x0cEnum.__new__\x94\x93\x94utRh\x0b\x85\x94R\x94.'

Thanks @pierreglaser!