From 3a3b6e33d6dfd255ad8f9c190e8209eb876b8280 Mon Sep 17 00:00:00 2001 From: Ammar Ahmad Awan Date: Wed, 14 Apr 2021 11:00:56 -0700 Subject: [PATCH 01/13] Add transformers tests https://github.com/microsoft/DeepSpeed/issues/937 --- .github/workflows/main.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bae175ab7fd0..c5c6016af567 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,3 +49,11 @@ jobs: run: | if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi TORCH_EXTENSIONS_DIR=./torch-extensions pytest --durations=0 --forked --verbose tests/unit/ + + - name: Transformer tests + run: | + if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi + git clone https://github.com/huggingface/transformers + cd transformers + pip install -e .[testing] + TORCH_EXTENSIONS_DIR=./torch-extensions RUN_SLOW=1 pytest --durations=0 --forked --verbose tests/deepspeed From 2503994c139d4f87fcfda06a0194f66c2d015b54 Mon Sep 17 00:00:00 2001 From: Ammar Ahmad Awan Date: Wed, 14 Apr 2021 11:04:33 -0700 Subject: [PATCH 02/13] Fix format. --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c5c6016af567..c4b35caa6020 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,7 +49,7 @@ jobs: run: | if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi TORCH_EXTENSIONS_DIR=./torch-extensions pytest --durations=0 --forked --verbose tests/unit/ - + - name: Transformer tests run: | if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi From 3a2bfc28661c36975c283f7f767e42138314ddee Mon Sep 17 00:00:00 2001 From: Ammar Ahmad Awan Date: Wed, 14 Apr 2021 12:35:03 -0700 Subject: [PATCH 03/13] Update .github/workflows/main.yml Co-authored-by: Stas Bekman --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c4b35caa6020..086d9668abae 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,7 +50,7 @@ jobs: if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi TORCH_EXTENSIONS_DIR=./torch-extensions pytest --durations=0 --forked --verbose tests/unit/ - - name: Transformer tests + - name: HF transformers tests run: | if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi git clone https://github.com/huggingface/transformers From f5f364073919b3a394350576adfd72ce6715ab30 Mon Sep 17 00:00:00 2001 From: Ammar Ahmad Awan Date: Wed, 14 Apr 2021 12:35:12 -0700 Subject: [PATCH 04/13] Update .github/workflows/main.yml Co-authored-by: Stas Bekman --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 086d9668abae..c494fa05b189 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -55,5 +55,6 @@ jobs: if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi git clone https://github.com/huggingface/transformers cd transformers - pip install -e .[testing] + pip install -e . + pip install sacrebleu rouge-score nltk parameterized psutil datasets TORCH_EXTENSIONS_DIR=./torch-extensions RUN_SLOW=1 pytest --durations=0 --forked --verbose tests/deepspeed From 2fa5c0de8c81bbce7a4f826f0f9fb8bb827d7189 Mon Sep 17 00:00:00 2001 From: Ammar Ahmad Awan Date: Mon, 19 Apr 2021 14:54:42 -0700 Subject: [PATCH 05/13] Update torch16.yml --- .github/workflows/torch16.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/torch16.yml b/.github/workflows/torch16.yml index cb7a375d2a53..54923de4a339 100644 --- a/.github/workflows/torch16.yml +++ b/.github/workflows/torch16.yml @@ -42,3 +42,12 @@ jobs: run: | if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi TORCH_EXTENSIONS_DIR=./torch-extensions pytest --durations=0 --forked --verbose -x tests/unit/ + - name: HF transformers tests + run: | + if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi + git clone https://github.com/huggingface/transformers + cd transformers + pip install -e . + pip install sacrebleu rouge-score nltk parameterized psutil datasets + TORCH_EXTENSIONS_DIR=./torch-extensions RUN_SLOW=1 pytest --durations=0 --forked --verbose tests/deepspeed + From 35171d4f0b5d3fe5bd3bce8da3f4ae411499cf85 Mon Sep 17 00:00:00 2001 From: Jeff Rasley Date: Fri, 24 Sep 2021 10:35:47 -0700 Subject: [PATCH 06/13] fix syntax and update example req install --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f20c08e8c812..bdebe95e8ddc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -70,7 +70,7 @@ jobs: runs-on: [self-hosted, nvidia, torch18, v100] steps: - -uses: actions/checkout@v2 + - uses: actions/checkout@v2 - name: environment run: | @@ -92,5 +92,5 @@ jobs: git rev-parse --short HEAD cd transformers pip install . - pip install sacrebleu rouge-score nltk parameterized psutil datasets + find examples/pytorch -name requirements.txt -exec pip install -r {} \ TORCH_EXTENSIONS_DIR=./torch-extensions RUN_SLOW=1 pytest --durations=0 --forked --verbose tests/deepspeed From f147c961f5d8e9982e0c21c3ecb869f9ea367ddc Mon Sep 17 00:00:00 2001 From: Jeff Rasley Date: Fri, 24 Sep 2021 10:38:15 -0700 Subject: [PATCH 07/13] Delete torch16.yml --- .github/workflows/torch16.yml | 55 ----------------------------------- 1 file changed, 55 deletions(-) delete mode 100755 .github/workflows/torch16.yml diff --git a/.github/workflows/torch16.yml b/.github/workflows/torch16.yml deleted file mode 100755 index 715f19120435..000000000000 --- a/.github/workflows/torch16.yml +++ /dev/null @@ -1,55 +0,0 @@ -# Unit test config for manual use on torch1.6 runners - -name: Torch16 - -# Controls when the action will run. -on: - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - # This workflow contains a single job called "build" - build: - # The type of runner that the job will run on - runs-on: [self-hosted, torch1.6] - - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 - - # Runs a single command using the runners shell - - name: environment - run: | - nvidia-smi - which python - python --version - which nvcc - nvcc --version - python -c "import torch; print('torch:', torch.__version__, torch)" - python -c "import torch; print('CUDA available:', torch.cuda.is_available())" - # Runs a set of commands using the runners shell - - name: Install deepspeed - run: | - pip install .[dev] - ds_report - - - name: Formatting checks - run: | - pre-commit run --all-files - - # Runs a set of commands using the runners shell - - name: Unit tests - run: | - if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi - TORCH_EXTENSIONS_DIR=./torch-extensions pytest --durations=0 --forked --verbose -x tests/unit/ - - name: HF transformers tests - run: | - if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi - git clone https://github.com/huggingface/transformers - cd transformers - pip install -e . - pip install sacrebleu rouge-score nltk parameterized psutil datasets - TORCH_EXTENSIONS_DIR=./torch-extensions RUN_SLOW=1 pytest --durations=0 --forked --verbose tests/deepspeed - From 1def932a4d2db8e460f8d8a467c2ef23c6330abf Mon Sep 17 00:00:00 2001 From: Jeff Rasley Date: Fri, 24 Sep 2021 10:41:18 -0700 Subject: [PATCH 08/13] fix find exec --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bdebe95e8ddc..c65f3752032c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -92,5 +92,5 @@ jobs: git rev-parse --short HEAD cd transformers pip install . - find examples/pytorch -name requirements.txt -exec pip install -r {} \ + find examples/pytorch -name requirements.txt -exec pip install -r {} \; TORCH_EXTENSIONS_DIR=./torch-extensions RUN_SLOW=1 pytest --durations=0 --forked --verbose tests/deepspeed From c2e5c3cb3010f445b444dee738ce6e3e2ee19847 Mon Sep 17 00:00:00 2001 From: Jeff Rasley Date: Fri, 24 Sep 2021 10:42:58 -0700 Subject: [PATCH 09/13] formatting --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c65f3752032c..41f3d2e03653 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -68,10 +68,10 @@ jobs: nv-transformers-v100: runs-on: [self-hosted, nvidia, torch18, v100] - + steps: - uses: actions/checkout@v2 - + - name: environment run: | nvidia-smi From 664b3722b160966eedcb34a47455c04c4320fc50 Mon Sep 17 00:00:00 2001 From: Jeff Rasley Date: Fri, 24 Sep 2021 11:44:52 -0700 Subject: [PATCH 10/13] exclude torch19 example req --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 41f3d2e03653..cd4b49916362 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -92,5 +92,6 @@ jobs: git rev-parse --short HEAD cd transformers pip install . - find examples/pytorch -name requirements.txt -exec pip install -r {} \; + # find all reqs, except for image-classification (it requires torch 1.9) + find examples/pytorch/ -path examples/pytorch/image-classification -prune -o -name requirements.txt -exec pip install -r {} \; TORCH_EXTENSIONS_DIR=./torch-extensions RUN_SLOW=1 pytest --durations=0 --forked --verbose tests/deepspeed From 37bff984529db3d14787441f0e3a6c6b356decab Mon Sep 17 00:00:00 2001 From: Jeff Rasley Date: Fri, 24 Sep 2021 12:40:30 -0700 Subject: [PATCH 11/13] force torch18+cu111 --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cd4b49916362..ee71f404205c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -53,6 +53,7 @@ jobs: python --version which nvcc nvcc --version + pip install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html python -c "import torch; print('torch:', torch.__version__, torch)" python -c "import torch; print('CUDA available:', torch.cuda.is_available())" - name: Install deepspeed @@ -79,6 +80,7 @@ jobs: python --version which nvcc nvcc --version + pip install torch==1.8.2+cu111 torchvision==0.9.2+cu111 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html python -c "import torch; print('torch:', torch.__version__, torch)" python -c "import torch; print('CUDA available:', torch.cuda.is_available())" - name: Install deepspeed From 633f84969f5bf6cc11416e7b4705b6f3e14f32b4 Mon Sep 17 00:00:00 2001 From: Jeff Rasley Date: Fri, 24 Sep 2021 12:44:36 -0700 Subject: [PATCH 12/13] update find command for reqs --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ee71f404205c..12b3b9758e5e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -94,6 +94,6 @@ jobs: git rev-parse --short HEAD cd transformers pip install . - # find all reqs, except for image-classification (it requires torch 1.9) - find examples/pytorch/ -path examples/pytorch/image-classification -prune -o -name requirements.txt -exec pip install -r {} \; - TORCH_EXTENSIONS_DIR=./torch-extensions RUN_SLOW=1 pytest --durations=0 --forked --verbose tests/deepspeed + # find reqs used in ds integration tests + find examples/pytorch -regextype posix-egrep -regex '.*(language-modeling|question-answering|summarization|text-classification|translation).*/requirements.txt' -exec pip install -r {} \; + TORCH_EXTENSIONS_DIR=./torch-extensions RUN_SLOW=1 pytest --durations=0 --verbose tests/deepspeed From cd982a36cc7290afa0ef95ee93a3e1789501e242 Mon Sep 17 00:00:00 2001 From: Jeff Rasley Date: Fri, 24 Sep 2021 14:24:12 -0700 Subject: [PATCH 13/13] install hf with testing extra --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 12b3b9758e5e..69e3da34e05f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -93,7 +93,7 @@ jobs: git clone https://github.com/huggingface/transformers git rev-parse --short HEAD cd transformers - pip install . + pip install .[testing] # find reqs used in ds integration tests find examples/pytorch -regextype posix-egrep -regex '.*(language-modeling|question-answering|summarization|text-classification|translation).*/requirements.txt' -exec pip install -r {} \; TORCH_EXTENSIONS_DIR=./torch-extensions RUN_SLOW=1 pytest --durations=0 --verbose tests/deepspeed