Cloud Pickle cannot serialize some objects.

[arita37]

I use map to execute some code.

############## Testing of IPyrallel on DEAP ###################################
creator.create("FitnessMin", base.Fitness, weights=(-1.0,))
creator.create("Individual", gp.PrimitiveTree, fitness=creator.FitnessMin)

toolbox = base.Toolbox()

**#Using Parallell Processing
import ipyparallel as ipp, time
rc= ipp.Client()
# pool = rc.load_balanced_view()
rc[:].use_cloudpickle()
pool= rc[:]
toolbox.register("map", pool.map)**


toolbox.register("expr", gp.genHalfAndHalf, pset=pset, min_=1, max_=2)
toolbox.register("individual", tools.initIterate, creator.Individual, toolbox.expr)
toolbox.register("population", tools.initRepeat, list, toolbox.individual)
toolbox.register("compile", gp.compile, pset=pset)


def evalSymbReg(individual, points):
 func = toolbox.compile(expr=individual) # Transform the tree expression in a callable function
 # and the real function : x**4 + x**3 + x**2 + x
 sqerrors = ((func(x) - x**4 - x**3 - x**2 - x)**2 for x in points)
 return math.fsum(sqerrors) / len(points),


toolbox.register("evaluate", evalSymbReg, points=[x/10. for x in range(-10,10)])
toolbox.register("select", tools.selTournament, tournsize=3)
toolbox.register("mate", gp.cxOnePoint)
toolbox.register("expr_mut", gp.genFull, min_=0, max_=2)
toolbox.register("mutate", gp.mutUniform, expr=toolbox.expr_mut, pset=pset)

toolbox.decorate("mate", gp.staticLimit(key=operator.attrgetter("height"), max_value=17))
toolbox.decorate("mutate", gp.staticLimit(key=operator.attrgetter("height"), max_value=17))

def main():
 random.seed(318)

 pop = toolbox.population(n=300)
 hof = tools.HallOfFame(1)
 
 stats_fit = tools.Statistics(lambda ind: ind.fitness.values)
 stats_size = tools.Statistics(len)
 mstats = tools.MultiStatistics(fitness=stats_fit, size=stats_size)
 mstats.register("avg", np.mean)
 mstats.register("std", np.std)
 mstats.register("min", np.min)
 mstats.register("max", np.max)

 pop, log = algorithms.eaSimple(pop, toolbox, 0.5, 0.1, 40, stats=mstats,
 halloffame=hof, verbose=True)
 # print log
 return pop, log, hof

I got this error :

---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-11-978da9be5b87> in <module>()
 1 if __name__ == "__main__":
----> 2 pop, log, hof= main()

<ipython-input-10-6ff69ab06682> in main()
 15 
 16 pop, log = algorithms.eaSimple(pop, toolbox, 0.5, 0.1, 40, stats=mstats,
---> 17 halloffame=hof, verbose=True)
 18 # print log
 19 return pop, log, hof

D:\_devs\Python01\Anaconda27\lib\site-packages\deap\algorithms.pyc in eaSimple(population, toolbox, cxpb, mutpb, ngen, stats, halloffame, verbose)
 145 # Evaluate the individuals with an invalid fitness
 146 invalid_ind = [ind for ind in population if not ind.fitness.valid]
--> 147 fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)
 148 for ind, fit in zip(invalid_ind, fitnesses):
 149 ind.fitness.values = fit

<decorator-gen-141> in map(self, f, *sequences, **kwargs)

D:\_devs\Python01\Anaconda27\lib\site-packages\ipyparallel\client\view.pyc in sync_results(f, self, *args, **kwargs)
 48 self._in_sync_results = True
 49 try:
---> 50 ret = f(self, *args, **kwargs)
 51 finally:
 52 self._in_sync_results = False

D:\_devs\Python01\Anaconda27\lib\site-packages\ipyparallel\client\view.pyc in map(self, f, *sequences, **kwargs)
 613 assert len(sequences) > 0, "must have some sequences to map onto!"
 614 pf = ParallelFunction(self, f, block=block, **kwargs)
--> 615 return pf.map(*sequences)
 616 
 617 @sync_results

D:\_devs\Python01\Anaconda27\lib\site-packages\ipyparallel\client\remotefunction.pyc in map(self, *sequences)
 283 and mismatched sequence lengths will be padded with None.
 284 """
--> 285 return self(*sequences, __ipp_mapping=True)
 286 
 287 __all__ = ['remote', 'parallel', 'RemoteFunction', 'ParallelFunction']

<decorator-gen-131> in __call__(self, *sequences, **kwargs)

D:\_devs\Python01\Anaconda27\lib\site-packages\ipyparallel\client\remotefunction.pyc in sync_view_results(f, self, *args, **kwargs)
 74 view = self.view
 75 if view._in_sync_results:
---> 76 return f(self, *args, **kwargs)
 77 view._in_sync_results = True
 78 try:

D:\_devs\Python01\Anaconda27\lib\site-packages\ipyparallel\client\remotefunction.pyc in __call__(self, *sequences, **kwargs)
 257 view = self.view if balanced else client[t]
 258 with view.temp_flags(block=False, **self.flags):
--> 259 ar = view.apply(f, *args)
 260 ar.owner = False
 261 

D:\_devs\Python01\Anaconda27\lib\site-packages\ipyparallel\client\view.pyc in apply(self, f, *args, **kwargs)
 209 ``f(*args, **kwargs)``.
 210 """
--> 211 return self._really_apply(f, args, kwargs)
 212 
 213 def apply_async(self, f, *args, **kwargs):

<decorator-gen-140> in _really_apply(self, f, args, kwargs, targets, block, track)

D:\_devs\Python01\Anaconda27\lib\site-packages\ipyparallel\client\view.pyc in sync_results(f, self, *args, **kwargs)
 48 self._in_sync_results = True
 49 try:
---> 50 ret = f(self, *args, **kwargs)
 51 finally:
 52 self._in_sync_results = False

<decorator-gen-139> in _really_apply(self, f, args, kwargs, targets, block, track)

D:\_devs\Python01\Anaconda27\lib\site-packages\ipyparallel\client\view.pyc in save_ids(f, self, *args, **kwargs)
 33 n_previous = len(self.client.history)
 34 try:
---> 35 ret = f(self, *args, **kwargs)
 36 finally:
 37 nmsgs = len(self.client.history) - n_previous

D:\_devs\Python01\Anaconda27\lib\site-packages\ipyparallel\client\view.pyc in _really_apply(self, f, args, kwargs, targets, block, track)
 555 for ident in _idents:
 556 future = self.client.send_apply_request(self._socket, f, args, kwargs, track=track,
--> 557 ident=ident)
 558 futures.append(future)
 559 if track:

D:\_devs\Python01\Anaconda27\lib\site-packages\ipyparallel\client\client.pyc in send_apply_request(self, socket, f, args, kwargs, metadata, track, ident)
 1387 bufs = serialize.pack_apply_message(f, args, kwargs,
 1388 buffer_threshold=self.session.buffer_threshold,
-> 1389 item_threshold=self.session.item_threshold,
 1390 )
 1391 

D:\_devs\Python01\Anaconda27\lib\site-packages\ipyparallel\serialize\serialize.pyc in pack_apply_message(f, args, kwargs, buffer_threshold, item_threshold)
 164 
 165 arg_bufs = list(chain.from_iterable(
--> 166 serialize_object(arg, buffer_threshold, item_threshold) for arg in args))
 167 
 168 kw_keys = sorted(kwargs.keys())

D:\_devs\Python01\Anaconda27\lib\site-packages\ipyparallel\serialize\serialize.pyc in <genexpr>((arg,))
 164 
 165 arg_bufs = list(chain.from_iterable(
--> 166 serialize_object(arg, buffer_threshold, item_threshold) for arg in args))
 167 
 168 kw_keys = sorted(kwargs.keys())

D:\_devs\Python01\Anaconda27\lib\site-packages\ipyparallel\serialize\serialize.pyc in serialize_object(obj, buffer_threshold, item_threshold)
 110 buffers.extend(_extract_buffers(cobj, buffer_threshold))
 111 
--> 112 buffers.insert(0, pickle.dumps(cobj, PICKLE_PROTOCOL))
 113 return buffers
 114 

D:\_devs\Python01\Anaconda27\lib\site-packages\cloudpickle\cloudpickle.pyc in dumps(obj, protocol)
 627 
 628 cp = CloudPickler(file,protocol)
--> 629 cp.dump(obj)
 630 
 631 return file.getvalue()

D:\_devs\Python01\Anaconda27\lib\site-packages\cloudpickle\cloudpickle.pyc in dump(self, obj)
 105 self.inject_addons()
 106 try:
--> 107 return Pickler.dump(self, obj)
 108 except RuntimeError as e:
 109 if 'recursion' in e.args[0]:

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in dump(self, obj)
 222 if self.proto >= 2:
 223 self.write(PROTO + chr(self.proto))
--> 224 self.save(obj)
 225 self.write(STOP)
 226 

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in save(self, obj)
 329 
 330 # Save the reduce() output and finally memoize the object
--> 331 self.save_reduce(obj=obj, *rv)
 332 
 333 def persistent_id(self, obj):

D:\_devs\Python01\Anaconda27\lib\site-packages\cloudpickle\cloudpickle.pyc in save_reduce(self, func, args, state, listitems, dictitems, obj)
 527 else:
 528 save(func)
--> 529 save(args)
 530 write(pickle.REDUCE)
 531 

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in save_tuple(self, obj)
 552 if n <= 3 and proto >= 2:
 553 for element in obj:
--> 554 save(element)
 555 # Subtle. Same as in the big comment below.
 556 if id(obj) in memo:

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

D:\_devs\Python01\Anaconda27\lib\site-packages\cloudpickle\cloudpickle.pyc in save_function(self, obj, name)
 203 or getattr(obj.__code__, 'co_filename', None) == '<stdin>'
 204 or themodule is None):
--> 205 self.save_function_tuple(obj)
 206 return
 207 else:

D:\_devs\Python01\Anaconda27\lib\site-packages\cloudpickle\cloudpickle.pyc in save_function_tuple(self, func)
 251 
 252 # save the rest of the func data needed by _fill_function
--> 253 save(f_globals)
 254 save(defaults)
 255 save(dct)

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in save_dict(self, obj)
 653 
 654 self.memoize(obj)
--> 655 self._batch_setitems(obj.iteritems())
 656 
 657 dispatch[DictionaryType] = save_dict

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in _batch_setitems(self, items)
 685 for k, v in tmp:
 686 save(k)
--> 687 save(v)
 688 write(SETITEMS)
 689 elif n:

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in save(self, obj)
 329 
 330 # Save the reduce() output and finally memoize the object
--> 331 self.save_reduce(obj=obj, *rv)
 332 
 333 def persistent_id(self, obj):

D:\_devs\Python01\Anaconda27\lib\site-packages\cloudpickle\cloudpickle.pyc in save_reduce(self, func, args, state, listitems, dictitems, obj)
 545 
 546 if state is not None:
--> 547 save(state)
 548 write(pickle.BUILD)
 549 

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in save_dict(self, obj)
 653 
 654 self.memoize(obj)
--> 655 self._batch_setitems(obj.iteritems())
 656 
 657 dispatch[DictionaryType] = save_dict

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in _batch_setitems(self, items)
 685 for k, v in tmp:
 686 save(k)
--> 687 save(v)
 688 write(SETITEMS)
 689 elif n:

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in save(self, obj)
 329 
 330 # Save the reduce() output and finally memoize the object
--> 331 self.save_reduce(obj=obj, *rv)
 332 
 333 def persistent_id(self, obj):

D:\_devs\Python01\Anaconda27\lib\site-packages\cloudpickle\cloudpickle.pyc in save_reduce(self, func, args, state, listitems, dictitems, obj)
 545 
 546 if state is not None:
--> 547 save(state)
 548 write(pickle.BUILD)
 549 

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in save_tuple(self, obj)
 566 write(MARK)
 567 for element in obj:
--> 568 save(element)
 569 
 570 if id(obj) in memo:

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in save_dict(self, obj)
 653 
 654 self.memoize(obj)
--> 655 self._batch_setitems(obj.iteritems())
 656 
 657 dispatch[DictionaryType] = save_dict

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in _batch_setitems(self, items)
 685 for k, v in tmp:
 686 save(k)
--> 687 save(v)
 688 write(SETITEMS)
 689 elif n:

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in save(self, obj)
 329 
 330 # Save the reduce() output and finally memoize the object
--> 331 self.save_reduce(obj=obj, *rv)
 332 
 333 def persistent_id(self, obj):

D:\_devs\Python01\Anaconda27\lib\site-packages\cloudpickle\cloudpickle.pyc in save_reduce(self, func, args, state, listitems, dictitems, obj)
 545 
 546 if state is not None:
--> 547 save(state)
 548 write(pickle.BUILD)
 549 

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in save_dict(self, obj)
 653 
 654 self.memoize(obj)
--> 655 self._batch_setitems(obj.iteritems())
 656 
 657 dispatch[DictionaryType] = save_dict

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in _batch_setitems(self, items)
 685 for k, v in tmp:
 686 save(k)
--> 687 save(v)
 688 write(SETITEMS)
 689 elif n:

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in save(self, obj)
 284 f = self.dispatch.get(t)
 285 if f:
--> 286 f(self, obj) # Call unbound method with explicit self
 287 return
 288 

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in save_dict(self, obj)
 653 
 654 self.memoize(obj)
--> 655 self._batch_setitems(obj.iteritems())
 656 
 657 dispatch[DictionaryType] = save_dict

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in _batch_setitems(self, items)
 684 write(MARK)
 685 for k, v in tmp:
--> 686 save(k)
 687 save(v)
 688 write(SETITEMS)

D:\_devs\Python01\Anaconda27\lib\pickle.pyc in save(self, obj)
 304 reduce = getattr(obj, "__reduce_ex__", None)
 305 if reduce:
--> 306 rv = reduce(self.proto)
 307 else:
 308 reduce = getattr(obj, "__reduce__", None)

TypeError: can't pickle member_descriptor objects

[jakirkham]

Are you able to reduce this to an MRE @arita37?

[pierreglaser]

The error that the traceback shows should not appear anymore thanks to #262. @arita37 can you try running your code again?

[jakirkham]

I think we can just go ahead and close this, @pierreglaser.