Cannot pickle certain type hints

[robertnishihara]

> python --version
Python 2.7.11 :: Anaconda 4.0.0 (x86_64)

Also

>>> cloudpickle.__version__
'0.1.1'

Hi, I'm trying to pickle some stuff in the typing module. I'm curious if there are fundamental limitations here or if this is out of scope for cloudpickle. Thanks for your help!

from typing import List, Callable
from cloudpickle import loads, dumps

This works.

>>> List
typing.List<~T>

>>> loads(dumps(List))
typing.List<~T>

This seems to lose some information.

>>> Callable[[int, str], float]
typing.Callable[[int, str], float]

>>> loads(dumps(Callable[[int, str], float]))
typing.Callable

This doesn't work

>>> dumps(List[int])
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-15-f02da844db1c> in <module>()
----> 1 dumps(List[int])

/Users/rkn/anaconda/lib/python2.7/site-packages/cloudpickle/cloudpickle.pyc in dumps(obj, protocol)
 600 
 601 cp = CloudPickler(file,protocol)
--> 602 cp.dump(obj)
 603 
 604 return file.getvalue()

/Users/rkn/anaconda/lib/python2.7/site-packages/cloudpickle/cloudpickle.pyc in dump(self, obj)
 105 self.inject_addons()
 106 try:
--> 107 return Pickler.dump(self, obj)
 108 except RuntimeError as e:
 109 if 'recursion' in e.args[0]:

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in dump(self, obj)
 222 if self.proto >= 2:
 223 self.write(PROTO + chr(self.proto))
--> 224 self.save(obj)
 225 self.write(STOP)
 226 

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 298 issc = 0
 299 if issc:
--> 300 self.save_global(obj)
 301 return
 302 

/Users/rkn/anaconda/lib/python2.7/site-packages/cloudpickle/cloudpickle.pyc in save_global(self, obj, name, pack)
 351 d['__new__'] = obj.__new__
 352 
--> 353 self.save_reduce(typ, (obj.__name__, obj.__bases__, d), obj=obj)
 354 else:
 355 raise pickle.PicklingError("Can't pickle %r" % obj)

/Users/rkn/anaconda/lib/python2.7/site-packages/cloudpickle/cloudpickle.pyc in save_reduce(self, func, args, state, listitems, dictitems, obj)
 509 else:
 510 save(func)
--> 511 save(args)
 512 write(pickle.REDUCE)
 513 

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save_tuple(self, obj)
 552 if n <= 3 and proto >= 2:
 553 for element in obj:
--> 554 save(element)
 555 # Subtle. Same as in the big comment below.
 556 if id(obj) in memo:

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save_tuple(self, obj)
 552 if n <= 3 and proto >= 2:
 553 for element in obj:
--> 554 save(element)
 555 # Subtle. Same as in the big comment below.
 556 if id(obj) in memo:

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 298 issc = 0
 299 if issc:
--> 300 self.save_global(obj)
 301 return
 302 

/Users/rkn/anaconda/lib/python2.7/site-packages/cloudpickle/cloudpickle.pyc in save_global(self, obj, name, pack)
 351 d['__new__'] = obj.__new__
 352 
--> 353 self.save_reduce(typ, (obj.__name__, obj.__bases__, d), obj=obj)
 354 else:
 355 raise pickle.PicklingError("Can't pickle %r" % obj)

/Users/rkn/anaconda/lib/python2.7/site-packages/cloudpickle/cloudpickle.pyc in save_reduce(self, func, args, state, listitems, dictitems, obj)
 509 else:
 510 save(func)
--> 511 save(args)
 512 write(pickle.REDUCE)
 513 

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save_tuple(self, obj)
 552 if n <= 3 and proto >= 2:
 553 for element in obj:
--> 554 save(element)
 555 # Subtle. Same as in the big comment below.
 556 if id(obj) in memo:

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save_tuple(self, obj)
 552 if n <= 3 and proto >= 2:
 553 for element in obj:
--> 554 save(element)
 555 # Subtle. Same as in the big comment below.
 556 if id(obj) in memo:

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 298 issc = 0
 299 if issc:
--> 300 self.save_global(obj)
 301 return
 302 

/Users/rkn/anaconda/lib/python2.7/site-packages/cloudpickle/cloudpickle.pyc in save_global(self, obj, name, pack)
 351 d['__new__'] = obj.__new__
 352 
--> 353 self.save_reduce(typ, (obj.__name__, obj.__bases__, d), obj=obj)
 354 else:
 355 raise pickle.PicklingError("Can't pickle %r" % obj)

/Users/rkn/anaconda/lib/python2.7/site-packages/cloudpickle/cloudpickle.pyc in save_reduce(self, func, args, state, listitems, dictitems, obj)
 509 else:
 510 save(func)
--> 511 save(args)
 512 write(pickle.REDUCE)
 513 

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save_tuple(self, obj)
 552 if n <= 3 and proto >= 2:
 553 for element in obj:
--> 554 save(element)
 555 # Subtle. Same as in the big comment below.
 556 if id(obj) in memo:

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save_tuple(self, obj)
 566 write(MARK)
 567 for element in obj:
--> 568 save(element)
 569 
 570 if id(obj) in memo:

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 298 issc = 0
 299 if issc:
--> 300 self.save_global(obj)
 301 return
 302 

/Users/rkn/anaconda/lib/python2.7/site-packages/cloudpickle/cloudpickle.pyc in save_global(self, obj, name, pack)
 351 d['__new__'] = obj.__new__
 352 
--> 353 self.save_reduce(typ, (obj.__name__, obj.__bases__, d), obj=obj)
 354 else:
 355 raise pickle.PicklingError("Can't pickle %r" % obj)

/Users/rkn/anaconda/lib/python2.7/site-packages/cloudpickle/cloudpickle.pyc in save_reduce(self, func, args, state, listitems, dictitems, obj)
 509 else:
 510 save(func)
--> 511 save(args)
 512 write(pickle.REDUCE)
 513 

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save_tuple(self, obj)
 552 if n <= 3 and proto >= 2:
 553 for element in obj:
--> 554 save(element)
 555 # Subtle. Same as in the big comment below.
 556 if id(obj) in memo:

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save_tuple(self, obj)
 552 if n <= 3 and proto >= 2:
 553 for element in obj:
--> 554 save(element)
 555 # Subtle. Same as in the big comment below.
 556 if id(obj) in memo:

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 298 issc = 0
 299 if issc:
--> 300 self.save_global(obj)
 301 return
 302 

/Users/rkn/anaconda/lib/python2.7/site-packages/cloudpickle/cloudpickle.pyc in save_global(self, obj, name, pack)
 351 d['__new__'] = obj.__new__
 352 
--> 353 self.save_reduce(typ, (obj.__name__, obj.__bases__, d), obj=obj)
 354 else:
 355 raise pickle.PicklingError("Can't pickle %r" % obj)

/Users/rkn/anaconda/lib/python2.7/site-packages/cloudpickle/cloudpickle.pyc in save_reduce(self, func, args, state, listitems, dictitems, obj)
 509 else:
 510 save(func)
--> 511 save(args)
 512 write(pickle.REDUCE)
 513 

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save_tuple(self, obj)
 552 if n <= 3 and proto >= 2:
 553 for element in obj:
--> 554 save(element)
 555 # Subtle. Same as in the big comment below.
 556 if id(obj) in memo:

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save_dict(self, obj)
 653 
 654 self.memoize(obj)
--> 655 self._batch_setitems(obj.iteritems())
 656 
 657 dispatch[DictionaryType] = save_dict

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in _batch_setitems(self, items)
 685 for k, v in tmp:
 686 save(k)
--> 687 save(v)
 688 write(SETITEMS)
 689 elif n:

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 329 
 330 # Save the reduce() output and finally memoize the object
--> 331 self.save_reduce(obj=obj, *rv)
 332 
 333 def persistent_id(self, obj):

/Users/rkn/anaconda/lib/python2.7/site-packages/cloudpickle/cloudpickle.pyc in save_reduce(self, func, args, state, listitems, dictitems, obj)
 527 
 528 if state is not None:
--> 529 save(state)
 530 write(pickle.BUILD)
 531 

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save_dict(self, obj)
 653 
 654 self.memoize(obj)
--> 655 self._batch_setitems(obj.iteritems())
 656 
 657 dispatch[DictionaryType] = save_dict

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in _batch_setitems(self, items)
 685 for k, v in tmp:
 686 save(k)
--> 687 save(v)
 688 write(SETITEMS)
 689 elif n:

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

/Users/rkn/anaconda/lib/python2.7/site-packages/cloudpickle/cloudpickle.pyc in save_function(self, obj, name)
 197 klass = getattr(themodule, name, None)
 198 if klass is None or klass is not obj:
--> 199 self.save_function_tuple(obj)
 200 return
 201 

/Users/rkn/anaconda/lib/python2.7/site-packages/cloudpickle/cloudpickle.pyc in save_function_tuple(self, func)
 240 # save the rest of the func data needed by _fill_function
 241 save(f_globals)
--> 242 save(defaults)
 243 save(dct)
 244 write(pickle.TUPLE)

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save_tuple(self, obj)
 552 if n <= 3 and proto >= 2:
 553 for element in obj:
--> 554 save(element)
 555 # Subtle. Same as in the big comment below.
 556 if id(obj) in memo:

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 329 
 330 # Save the reduce() output and finally memoize the object
--> 331 self.save_reduce(obj=obj, *rv)
 332 
 333 def persistent_id(self, obj):

/Users/rkn/anaconda/lib/python2.7/site-packages/cloudpickle/cloudpickle.pyc in save_reduce(self, func, args, state, listitems, dictitems, obj)
 494 "args[0] from __newobj__ args has the wrong class")
 495 args = args[1:]
--> 496 save(cls)
 497 
 498 #Don't pickle transient entries

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

/Users/rkn/anaconda/lib/python2.7/site-packages/cloudpickle/cloudpickle.pyc in save_global(self, obj, name, pack)
 351 d['__new__'] = obj.__new__
 352 
--> 353 self.save_reduce(typ, (obj.__name__, obj.__bases__, d), obj=obj)
 354 else:
 355 raise pickle.PicklingError("Can't pickle %r" % obj)

/Users/rkn/anaconda/lib/python2.7/site-packages/cloudpickle/cloudpickle.pyc in save_reduce(self, func, args, state, listitems, dictitems, obj)
 509 else:
 510 save(func)
--> 511 save(args)
 512 write(pickle.REDUCE)
 513 

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save_tuple(self, obj)
 552 if n <= 3 and proto >= 2:
 553 for element in obj:
--> 554 save(element)
 555 # Subtle. Same as in the big comment below.
 556 if id(obj) in memo:

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save_dict(self, obj)
 653 
 654 self.memoize(obj)
--> 655 self._batch_setitems(obj.iteritems())
 656 
 657 dispatch[DictionaryType] = save_dict

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in _batch_setitems(self, items)
 685 for k, v in tmp:
 686 save(k)
--> 687 save(v)
 688 write(SETITEMS)
 689 elif n:

/Users/rkn/anaconda/lib/python2.7/pickle.pyc in save(self, obj)
 304 reduce = getattr(obj, "__reduce_ex__", None)
 305 if reduce:
--> 306 rv = reduce(self.proto)
 307 else:
 308 reduce = getattr(obj, "__reduce__", None)

TypeError: can't pickle wrapper_descriptor objects

[nareshsankapelly]

I'm also seeing similar error.

[bdod6]

I'm trying to test Ray on a RF classifier, and I'm getting the same error from the code below:

@ray.remote
def run_classifiers(x, y, x_test, y_test):
 rnd_clf = RandomForestClassifier(random_state=42)
 rnd_clf.fit(x, y)
 y_pred = clf.predict(x_test)
 return accuracy_score(y_test, y_pred)

clf_results = run_classifiers.remote(X_train, y_train, X_test, y_test)

[robertnishihara]

@bdod6, can you post the full code so that we can reproduce the problem easily? Also, can you post the error message?

It may make sense to migrate this issue over to the Ray repository.

[bdod6]

Figured it out. I was passing a pandas dataframe to the remote function. It went away after I changed it to numpy array. Thanks!

[pcmoritz]

If you use a Ray version after ray-project/ray@0684258, passing dataframes to remote functions should now work (it was merged very recently).

…

On Thu, Oct 12, 2017 at 2:47 PM, bdod6 ***@***.***> wrote: Figured it out. I was passing a pandas dataframe to the remote function. It went away after I changed it to numpy array. Thanks! — You are receiving this because you are subscribed to this thread. Reply to this email directly, view it on GitHub <#63 (comment)>, or mute the thread <https://github.com/notifications/unsubscribe-auth/AAG6pP1hHmC4b4m2GywhLnsgH_m_w1MZks5srojpgaJpZM4JQanV> .

[bdod6]

Awesome...thank you!

[pierreglaser]

OK some improvements in #262 make all examples in #63 (comment) pass. So I guess we can close this.

[jrbourbeau]

OK some improvements in #262 make all examples in #63 (comment) pass. So I guess we can close this.

Yep, I just confirmed this is still the case today. Should be good to close this issue 👍