Fix cloudpickle incompatibilities on early Python 3.5 versions

CHANGES.md

@@ -1,6 +1,10 @@
 1.4.1 (in development)
 ======================
 
+- Fix incompatibilities between cloudpickle 1.4.0 and Python 3.5.0/1/2
+ introduced by the new support of cloudpickle for pickling typing constructs.
+ ([issue #360](https://github.com/cloudpipe/cloudpickle/issues/360))
+
 - Restore compat with loading dynamic classes pickled with cloudpickle
  version 1.2.1 that would reference the `types.ClassType` attribute.
  ([PR #359](https://github.com/cloudpipe/cloudpickle/pull/359))

cloudpickle/cloudpickle.py

@@ -61,7 +61,7 @@
 import typing
 from enum import Enum
 
-from typing import Generic, Union, Tuple, Callable, ClassVar
+from typing import Generic, Union, Tuple, Callable
 from pickle import _Pickler as Pickler
 from pickle import _getattribute
 from io import BytesIO
@@ -73,6 +73,11 @@
 except ImportError:
  _typing_extensions = Literal = Final = None
 
+if sys.version_info >= (3, 5, 3):
+ from typing import ClassVar
+else: # pragma: no cover
+ ClassVar = None
+
 
 # cloudpickle is meant for inter process communication: we expect all
 # communicating processes to run the same Python version hence we favor
@@ -432,10 +437,24 @@ def _extract_class_dict(cls):
 if sys.version_info[:2] < (3, 7): # pragma: no branch
  def _is_parametrized_type_hint(obj):
  # This is very cheap but might generate false positives.
- origin = getattr(obj, '__origin__', None) # typing Constructs
- values = getattr(obj, '__values__', None) # typing_extensions.Literal
- type_ = getattr(obj, '__type__', None) # typing_extensions.Final
- return origin is not None or values is not None or type_ is not None
+ # general typing Constructs
+ is_typing = getattr(obj, '__origin__', None) is not None
+
+ # typing_extensions.Literal
+ is_litteral = getattr(obj, '__values__', None) is not None
+
+ # typing_extensions.Final
+ is_final = getattr(obj, '__type__', None) is not None
+
+ # typing.Union/Tuple for old Python 3.5
+ is_union = getattr(obj, '__union_params__', None) is not None
+ is_tuple = getattr(obj, '__tuple_params__', None) is not None
+ is_callable = (
+ getattr(obj, '__result__', None) is not None and
+ getattr(obj, '__args__', None) is not None
+ )
+ return any((is_typing, is_litteral, is_final, is_union, is_tuple,
+ is_callable))
 
  def _create_parametrized_type_hint(origin, args):
  return origin[args]
@@ -971,14 +990,40 @@ def _save_parametrized_type_hint(self, obj):
  initargs = (Final, obj.__type__)
  elif type(obj) is type(ClassVar):
  initargs = (ClassVar, obj.__type__)
- elif type(obj) in [type(Union), type(Tuple), type(Generic)]:
- initargs = (obj.__origin__, obj.__args__)
+ elif type(obj) is type(Generic):
+ parameters = obj.__parameters__
+ if len(obj.__parameters__) > 0:
+ # in early Python 3.5, __parameters__ was sometimes
+ # preferred to __args__
+ initargs = (obj.__origin__, parameters)
+ else:
+ initargs = (obj.__origin__, obj.__args__)
+ elif type(obj) is type(Union):
+ if sys.version_info < (3, 5, 3): # pragma: no cover
+ initargs = (Union, obj.__union_params__)
+ else:
+ initargs = (Union, obj.__args__)
+ elif type(obj) is type(Tuple):
+ if sys.version_info < (3, 5, 3): # pragma: no cover
+ initargs = (Tuple, obj.__tuple_params__)
+ else:
+ initargs = (Tuple, obj.__args__)
  elif type(obj) is type(Callable):
- args = obj.__args__
- if args[0] is Ellipsis:
- initargs = (obj.__origin__, args)
+ if sys.version_info < (3, 5, 3): # pragma: no cover
+ args = obj.__args__
+ result = obj.__result__
+ if args != Ellipsis:
+ if isinstance(args, tuple):
+ args = list(args)
+ else:
+ args = [args]
  else:
- initargs = (obj.__origin__, (list(args[:-1]), args[-1]))
+ (*args, result) = obj.__args__
+ if len(args) == 1 and args[0] is Ellipsis:
+ args = Ellipsis
+ else:
+ args = list(args)
+ initargs = (Callable, (args, result))
  else: # pragma: no cover
  raise pickle.PicklingError(
  "Cloudpickle Error: Unknown type {}".format(type(obj))
@@ -1301,14 +1346,23 @@ def _make_typevar(name, bound, constraints, covariant, contravariant,
  name, *constraints, bound=bound,
  covariant=covariant, contravariant=contravariant
  )
- return _lookup_class_or_track(class_tracker_id, tv)
+ if class_tracker_id is not None:
+ return _lookup_class_or_track(class_tracker_id, tv)
+ else: # pragma: nocover
+ # Only for Python 3.5.3 compat.
+ return tv
 
 
 def _decompose_typevar(obj):
+ try:
+ class_tracker_id = _get_or_create_tracker_id(obj)
+ except TypeError: # pragma: nocover
+ # TypeVar instances are not weakref-able in Python 3.5.3
+ class_tracker_id = None
  return (
  obj.__name__, obj.__bound__, obj.__constraints__,
  obj.__covariant__, obj.__contravariant__,
-_get_or_create_tracker_id(obj),
+class_tracker_id,
  )
 
 

tests/cloudpickle_test.py

@@ -2047,6 +2047,9 @@ def test_pickle_dynamic_typevar(self):
  for attr in attr_list:
  assert getattr(T, attr) == getattr(depickled_T, attr)
 
+ @pytest.mark.skipif(
+ sys.version_info[:3] == (3, 5, 3),
+ reason="TypeVar instances are not weakref-able in Python 3.5.3")
  def test_pickle_dynamic_typevar_tracking(self):
  T = typing.TypeVar("T")
  T2 = subprocess_pickle_echo(T, protocol=self.protocol)
@@ -2081,20 +2084,32 @@ class C(typing.Generic[T]):
 
  with subprocess_worker(protocol=self.protocol) as worker:
 
- def check_generic(generic, origin, type_value):
+ def check_generic(generic, origin, type_value, use_args):
  assert generic.__origin__ is origin
- assert len(generic.__args__) == 1
- assert generic.__args__[0] is type_value
 
- assert len(origin.__orig_bases__) == 1
- ob = origin.__orig_bases__[0]
- assert ob.__origin__ is typing.Generic
+ if sys.version_info >= (3, 5, 3):
+ assert len(origin.__orig_bases__) == 1
+ ob = origin.__orig_bases__[0]
+ assert ob.__origin__ is typing.Generic
+ else: # Python 3.5.[0-1-2], pragma: no cover
+ assert len(origin.__bases__) == 1
+ ob = origin.__bases__[0]
+
+ if use_args:
+ assert len(generic.__args__) == 1
+ assert generic.__args__[0] is type_value
+ else:
+ assert len(generic.__parameters__) == 1
+ assert generic.__parameters__[0] is type_value
  assert len(ob.__parameters__) == 1
 
  return "ok"
 
- assert check_generic(C[int], C, int) == "ok"
- assert worker.run(check_generic, C[int], C, int) == "ok"
+ # backward-compat for old Python 3.5 versions that sometimes relies
+ # on __parameters__
+ use_args = getattr(C[int], '__args__', ()) != ()
+ assert check_generic(C[int], C, int, use_args) == "ok"
+ assert worker.run(check_generic, C[int], C, int, use_args) == "ok"
 
  def test_locally_defined_class_with_type_hints(self):
  with subprocess_worker(protocol=self.protocol) as worker:
@@ -2116,19 +2131,38 @@ def check_annotations(obj, expected_type):
  assert check_annotations(obj, type_) == "ok"
  assert worker.run(check_annotations, obj, type_) == "ok"
 
- def test_generic_extensions(self):
+ def test_generic_extensions_literal(self):
  typing_extensions = pytest.importorskip('typing_extensions')
 
-objs = [
-typing_extensions.Literal,
- typing_extensions.Final,
- typing_extensions.Literal['a'],
- typing_extensions.Final[int],
+def check_literal_equal(obj1, obj2):
+assert obj1.__values__ == obj2.__values__
+assert type(obj1) == type(obj2) == typing_extensions._LiteralMeta
+literal_objs = [
+ typing_extensions.Literal, typing_extensions.Literal['a']
  ]
+ for obj in literal_objs:
+ depickled_obj = pickle_depickle(obj, protocol=self.protocol)
+ if sys.version_info[:3] >= (3, 5, 3):
+ assert depickled_obj == obj
+ else:
+ # __eq__ does not work for Literal objects in early Python 3.5
+ check_literal_equal(obj, depickled_obj)
+
+ def test_generic_extensions_final(self):
+ typing_extensions = pytest.importorskip('typing_extensions')
+
+ def check_final_equal(obj1, obj2):
+ assert obj1.__type__ == obj2.__type__
+ assert type(obj1) == type(obj2) == typing_extensions._FinalMeta
+ final_objs = [typing_extensions.Final, typing_extensions.Final[int]]
 
- for obj in objs:
+ for obj in final_objs:
  depickled_obj = pickle_depickle(obj, protocol=self.protocol)
- assert depickled_obj == obj
+ if sys.version_info[:3] >= (3, 5, 3):
+ assert depickled_obj == obj
+ else:
+ # __eq__ does not work for Final objects in early Python 3.5
+ check_final_equal(obj, depickled_obj)
 
  def test_class_annotations(self):
  class C:
@@ -2182,21 +2216,26 @@ def _all_types_to_test():
  class C(typing.Generic[T]):
  pass
 
-return [
+types_to_test = [
  C, C[int],
- T, typing.Any, typing.NoReturn, typing.Optional,
- typing.Generic, typing.Union, typing.ClassVar,
+ T, typing.Any, typing.Optional,
+ typing.Generic, typing.Union,
  typing.Optional[int],
  typing.Generic[T],
  typing.Callable[[int], typing.Any],
  typing.Callable[..., typing.Any],
  typing.Callable[[], typing.Any],
  typing.Tuple[int, ...],
  typing.Tuple[int, C[int]],
- typing.ClassVar[C[int]],
  typing.List[int],
  typing.Dict[int, str],
  ]
+ if sys.version_info[:3] >= (3, 5, 3):
+ types_to_test.append(typing.ClassVar)
+ types_to_test.append(typing.ClassVar[C[int]])
+ if sys.version_info >= (3, 5, 4):
+ types_to_test.append(typing.NoReturn)
+ return types_to_test
 
 
 if __name__ == '__main__':