From b16c781b06fb33101e1d64f5bd87fac2d98a42b7 Mon Sep 17 00:00:00 2001 From: Samyam Rajbhandari Date: Tue, 8 Jun 2021 16:44:28 -0700 Subject: [PATCH 1/5] largest_partitioned_params calculation fix largest partitioned params was getting calculated incorrectly --- deepspeed/runtime/zero/stage3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepspeed/runtime/zero/stage3.py b/deepspeed/runtime/zero/stage3.py index cac206ed759b..ce4b49ac6094 100755 --- a/deepspeed/runtime/zero/stage3.py +++ b/deepspeed/runtime/zero/stage3.py @@ -852,7 +852,7 @@ def __init__(self, count = count + 1 #Largest partitioned param - largest_partitioned_param_numel = max(self.fp16_partitioned_groups_flat_numel) + largest_partitioned_param_numel = max([max(fp16_partitioned_group) for fp16_partitioned_group in self.fp16_partitioned_groups]) print_rank_0( f'Largest partitioned param numel = {largest_partitioned_param_numel}', force=True) From c1ec6e2bce809aa928f97cba9e04be576dd93647 Mon Sep 17 00:00:00 2001 From: Samyam Rajbhandari Date: Tue, 8 Jun 2021 16:51:32 -0700 Subject: [PATCH 2/5] Update stage3.py --- deepspeed/runtime/zero/stage3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepspeed/runtime/zero/stage3.py b/deepspeed/runtime/zero/stage3.py index ce4b49ac6094..b5ac0d2d925c 100755 --- a/deepspeed/runtime/zero/stage3.py +++ b/deepspeed/runtime/zero/stage3.py @@ -852,7 +852,7 @@ def __init__(self, count = count + 1 #Largest partitioned param - largest_partitioned_param_numel = max([max(fp16_partitioned_group) for fp16_partitioned_group in self.fp16_partitioned_groups]) + largest_partitioned_param_numel = max([max([tensor.numel for tensor in fp16_partitioned_group]) for fp16_partitioned_group in self.fp16_partitioned_groups]) print_rank_0( f'Largest partitioned param numel = {largest_partitioned_param_numel}', force=True) From b52b333f385cff73751415e177bd7b63d6a07a73 Mon Sep 17 00:00:00 2001 From: Samyam Rajbhandari Date: Tue, 8 Jun 2021 16:52:23 -0700 Subject: [PATCH 3/5] Update stage3.py --- deepspeed/runtime/zero/stage3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepspeed/runtime/zero/stage3.py b/deepspeed/runtime/zero/stage3.py index b5ac0d2d925c..3089c9a4c9b3 100755 --- a/deepspeed/runtime/zero/stage3.py +++ b/deepspeed/runtime/zero/stage3.py @@ -852,7 +852,7 @@ def __init__(self, count = count + 1 #Largest partitioned param - largest_partitioned_param_numel = max([max([tensor.numel for tensor in fp16_partitioned_group]) for fp16_partitioned_group in self.fp16_partitioned_groups]) + largest_partitioned_param_numel = max([max([tensor.numel() for tensor in fp16_partitioned_group]) for fp16_partitioned_group in self.fp16_partitioned_groups]) print_rank_0( f'Largest partitioned param numel = {largest_partitioned_param_numel}', force=True) From b7e9e9bcac117db1da7ede51fe863c96545bf8d9 Mon Sep 17 00:00:00 2001 From: Samyam Rajbhandari Date: Wed, 9 Jun 2021 00:28:39 +0000 Subject: [PATCH 4/5] formatting fix --- deepspeed/runtime/zero/stage3.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deepspeed/runtime/zero/stage3.py b/deepspeed/runtime/zero/stage3.py index 3089c9a4c9b3..548c38a072c3 100755 --- a/deepspeed/runtime/zero/stage3.py +++ b/deepspeed/runtime/zero/stage3.py @@ -852,7 +852,10 @@ def __init__(self, count = count + 1 #Largest partitioned param - largest_partitioned_param_numel = max([max([tensor.numel() for tensor in fp16_partitioned_group]) for fp16_partitioned_group in self.fp16_partitioned_groups]) + largest_partitioned_param_numel = max([ + max([tensor.numel() for tensor in fp16_partitioned_group]) + for fp16_partitioned_group in self.fp16_partitioned_groups + ]) print_rank_0( f'Largest partitioned param numel = {largest_partitioned_param_numel}', force=True) From 307ccdfa0631de6db2b8e5972bde8010bfce16fb Mon Sep 17 00:00:00 2001 From: Samyam Rajbhandari Date: Wed, 9 Jun 2021 00:44:01 +0000 Subject: [PATCH 5/5] changing sub-group size default to 1e9 --- deepspeed/runtime/zero/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepspeed/runtime/zero/constants.py b/deepspeed/runtime/zero/constants.py index eaeb2a95ccd9..7beebe00e717 100755 --- a/deepspeed/runtime/zero/constants.py +++ b/deepspeed/runtime/zero/constants.py @@ -91,7 +91,7 @@ ZERO_OPTIMIZATION_OFFLOAD_OPTIMIZER_DEFAULT = None ZERO_OPTIMIZATION_SUB_GROUP_SIZE = 'sub_group_size' -ZERO_OPTIMIZATION_SUB_GROUP_SIZE_DEFAULT = 1000000000000 +ZERO_OPTIMIZATION_SUB_GROUP_SIZE_DEFAULT = 1000000000 #maximum number of parameters per GPU before releasing them ZERO_OPTIMIZATION_MAX_LIVE_PARAMETERS = 'stage3_max_live_parameters'