Skip to content

Commit 72716b1

Browse files
borsbors
committed
Auto merge of #144305 - ChrisDenton:win-free-disk-space, r=marcoieni
Free disk space on Windows 2025 runners I've managed to reduce the time deletion takes by: - Using powershell, which is generally faster for filesystem operations than msys2 - Performing deletions concurrently then waiting for them all to complete It still takes 2-10 mins but that's not too bad.
2 parents 919c409 + 43018d3 commit 72716b1

5 files changed

Lines changed: 316 additions & 274 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,14 +117,21 @@ jobs:
117117
with:
118118
fetch-depth: 2
119119

120-
# Free up disk space on Linux by removing preinstalled components that
120+
# Free up disk space on Linux and Windows by removing preinstalled components that
121121
# we do not need. We do this to enable some of the less resource
122122
# intensive jobs to run on free runners, which however also have
123123
# less disk space.
124124
- name: free up disk space
125125
run: src/ci/scripts/free-disk-space.sh
126126
if: matrix.free_disk
127127

128+
# If we don't need to free up disk space then just report how much space we have
129+
- name: print disk usage
130+
run: |
131+
echo "disk usage:"
132+
df -h
133+
if: matrix.free_disk == false
134+
128135
# Rust Log Analyzer can't currently detect the PR number of a GitHub
129136
# Actions build on its own, so a hint in the log message is needed to
130137
# point it in the right direction.

‎src/ci/github-actions/jobs.yml‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -31,20 +31,11 @@ runners:
3131
<<: *base-job
3232

3333
- &job-windows
34-
os: windows-2022
35-
<<: *base-job
36-
37-
# NOTE: windows-2025 has less disk space available than windows-2022,
38-
# because the D drive is missing.
39-
- &job-windows-25
4034
os: windows-2025
35+
free_disk: true
4136
<<: *base-job
4237

4338
- &job-windows-8c
44-
os: windows-2022-8core-32gb
45-
<<: *base-job
46-
47-
- &job-windows-25-8c
4839
os: windows-2025-8core-32gb
4940
<<: *base-job
5041

@@ -668,7 +659,7 @@ auto:
668659
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
669660
DIST_REQUIRE_ALL_TOOLS: 1
670661
CODEGEN_BACKENDS: llvm,cranelift
671-
<<: *job-windows-25-8c
662+
<<: *job-windows-8c
672663

673664
- name: dist-i686-msvc
674665
env:

‎src/ci/scripts/free-disk-space-linux.sh‎

Lines changed: 265 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,265 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
# Free disk space on Linux GitHub action runners
5+
# Script inspired by https://github.com/jlumbroso/free-disk-space
6+
7+
isX86() {
8+
local arch
9+
arch=$(uname -m)
10+
if [ "$arch"="x86_64" ];then
11+
return 0
12+
else
13+
return 1
14+
fi
15+
}
16+
17+
# Check if we're on a GitHub hosted runner.
18+
# In aws codebuild, the variable RUNNER_ENVIRONMENT is "self-hosted".
19+
isGitHubRunner() {
20+
# `:-` means "use the value of RUNNER_ENVIRONMENT if it exists, otherwise use an empty string".
21+
if [[ "${RUNNER_ENVIRONMENT:-}"=="github-hosted" ]];then
22+
return 0
23+
else
24+
return 1
25+
fi
26+
}
27+
28+
# print a line of the specified character
29+
printSeparationLine() {
30+
for((i =0; i <80; i++));do
31+
printf"%s""$1"
32+
done
33+
printf"\n"
34+
}
35+
36+
# compute available space
37+
# REF: https://unix.stackexchange.com/a/42049/60849
38+
# REF: https://stackoverflow.com/a/450821/408734
39+
getAvailableSpace() {
40+
df -a | awk 'NR > 1 {avail+=$4} END {print avail}'
41+
}
42+
43+
# make Kb human readable (assume the input is Kb)
44+
# REF: https://unix.stackexchange.com/a/44087/60849
45+
formatByteCount() {
46+
numfmt --to=iec-i --suffix=B --padding=7 "${1}000"
47+
}
48+
49+
# macro to output saved space
50+
printSavedSpace() {
51+
# Disk space before the operation
52+
local before=${1}
53+
local title=${2:-}
54+
55+
local after
56+
after=$(getAvailableSpace)
57+
local saved=$((after - before))
58+
59+
if [ "$saved"-lt 0 ];then
60+
echo"::warning::Saved space is negative: $saved. Using '0' as saved space."
61+
saved=0
62+
fi
63+
64+
echo""
65+
printSeparationLine "*"
66+
if [ -n"${title}" ];then
67+
echo"=> ${title}: Saved $(formatByteCount "$saved")"
68+
else
69+
echo"=> Saved $(formatByteCount "$saved")"
70+
fi
71+
printSeparationLine "*"
72+
echo""
73+
}
74+
75+
# macro to print output of df with caption
76+
printDF() {
77+
local caption=${1}
78+
79+
printSeparationLine "="
80+
echo"${caption}"
81+
echo""
82+
echo"$ df -h"
83+
echo""
84+
df -h
85+
printSeparationLine "="
86+
}
87+
88+
removeUnusedFilesAndDirs() {
89+
local to_remove=(
90+
"/usr/share/java"
91+
)
92+
93+
if isGitHubRunner;then
94+
to_remove+=(
95+
"/usr/local/aws-sam-cli"
96+
"/usr/local/doc/cmake"
97+
"/usr/local/julia"*
98+
"/usr/local/lib/android"
99+
"/usr/local/share/chromedriver-"*
100+
"/usr/local/share/chromium"
101+
"/usr/local/share/cmake-"*
102+
"/usr/local/share/edge_driver"
103+
"/usr/local/share/emacs"
104+
"/usr/local/share/gecko_driver"
105+
"/usr/local/share/icons"
106+
"/usr/local/share/powershell"
107+
"/usr/local/share/vcpkg"
108+
"/usr/local/share/vim"
109+
"/usr/share/apache-maven-"*
110+
"/usr/share/gradle-"*
111+
"/usr/share/kotlinc"
112+
"/usr/share/miniconda"
113+
"/usr/share/php"
114+
"/usr/share/ri"
115+
"/usr/share/swift"
116+
117+
# binaries
118+
"/usr/local/bin/azcopy"
119+
"/usr/local/bin/bicep"
120+
"/usr/local/bin/ccmake"
121+
"/usr/local/bin/cmake-"*
122+
"/usr/local/bin/cmake"
123+
"/usr/local/bin/cpack"
124+
"/usr/local/bin/ctest"
125+
"/usr/local/bin/helm"
126+
"/usr/local/bin/kind"
127+
"/usr/local/bin/kustomize"
128+
"/usr/local/bin/minikube"
129+
"/usr/local/bin/packer"
130+
"/usr/local/bin/phpunit"
131+
"/usr/local/bin/pulumi-"*
132+
"/usr/local/bin/pulumi"
133+
"/usr/local/bin/stack"
134+
135+
# Haskell runtime
136+
"/usr/local/.ghcup"
137+
138+
# Azure
139+
"/opt/az"
140+
"/usr/share/az_"*
141+
)
142+
143+
if [ -n"${AGENT_TOOLSDIRECTORY:-}" ];then
144+
# Environment variable set by GitHub Actions
145+
to_remove+=(
146+
"${AGENT_TOOLSDIRECTORY}"
147+
)
148+
else
149+
echo"::warning::AGENT_TOOLSDIRECTORY is not set. Skipping removal."
150+
fi
151+
else
152+
# Remove folders and files present in AWS CodeBuild
153+
to_remove+=(
154+
# binaries
155+
"/usr/local/bin/ecs-cli"
156+
"/usr/local/bin/eksctl"
157+
"/usr/local/bin/kubectl"
158+
159+
"${HOME}/.gradle"
160+
"${HOME}/.dotnet"
161+
"${HOME}/.goenv"
162+
"${HOME}/.phpenv"
163+
164+
)
165+
fi
166+
167+
forelementin"${to_remove[@]}";do
168+
if [ !-e"$element" ];then
169+
# The file or directory doesn't exist.
170+
# Maybe it was removed in a newer version of the runner or it's not present in a
171+
# specific architecture (e.g. ARM).
172+
echo"::warning::Directory or file $element does not exist, skipping."
173+
fi
174+
done
175+
176+
# Remove all files and directories at once to save time.
177+
sudo rm -rf "${to_remove[@]}"
178+
}
179+
180+
execAndMeasureSpaceChange() {
181+
local operation=${1}# Function to execute
182+
local title=${2}
183+
184+
local before
185+
before=$(getAvailableSpace)
186+
$operation
187+
188+
printSavedSpace "$before""$title"
189+
}
190+
191+
# Remove large packages
192+
# REF: https://github.com/apache/flink/blob/master/tools/azure-pipelines/free_disk_space.sh
193+
cleanPackages() {
194+
local packages=(
195+
'^aspnetcore-.*'
196+
'^dotnet-.*'
197+
'^llvm-.*'
198+
'^mongodb-.*'
199+
'firefox'
200+
'libgl1-mesa-dri'
201+
'mono-devel'
202+
'php.*'
203+
)
204+
205+
if isGitHubRunner;then
206+
packages+=(
207+
azure-cli
208+
)
209+
210+
if isX86;then
211+
packages+=(
212+
'google-chrome-stable'
213+
'google-cloud-cli'
214+
'google-cloud-sdk'
215+
'powershell'
216+
)
217+
fi
218+
else
219+
packages+=(
220+
'google-chrome-stable'
221+
)
222+
fi
223+
224+
sudo apt-get -qq remove -y --fix-missing "${packages[@]}"
225+
226+
sudo apt-get autoremove -y ||echo"::warning::The command [sudo apt-get autoremove -y] failed"
227+
sudo apt-get clean ||echo"::warning::The command [sudo apt-get clean] failed failed"
228+
}
229+
230+
# Remove Docker images.
231+
# Ubuntu 22 runners have docker images already installed.
232+
# They aren't present in ubuntu 24 runners.
233+
cleanDocker() {
234+
echo"=> Removing the following docker images:"
235+
sudo docker image ls
236+
echo"=> Removing docker images..."
237+
sudo docker image prune --all --force ||true
238+
}
239+
240+
# Remove Swap storage
241+
cleanSwap() {
242+
sudo swapoff -a ||true
243+
sudo rm -rf /mnt/swapfile ||true
244+
free -h
245+
}
246+
247+
# Display initial disk space stats
248+
249+
AVAILABLE_INITIAL=$(getAvailableSpace)
250+
251+
printDF "BEFORE CLEAN-UP:"
252+
echo""
253+
execAndMeasureSpaceChange cleanPackages "Unused packages"
254+
execAndMeasureSpaceChange cleanDocker "Docker images"
255+
execAndMeasureSpaceChange cleanSwap "Swap storage"
256+
execAndMeasureSpaceChange removeUnusedFilesAndDirs "Unused files and directories"
257+
258+
# Output saved space statistic
259+
echo""
260+
printDF "AFTER CLEAN-UP:"
261+
262+
echo""
263+
echo""
264+
265+
printSavedSpace "$AVAILABLE_INITIAL""Total saved"

‎src/ci/scripts/free-disk-space-windows.ps1‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Free disk space on Windows GitHub action runners.
2+
3+
$ErrorActionPreference='Stop'
4+
5+
Get-Volume|Out-String|Write-Output
6+
7+
$available=$(Get-Volume C).SizeRemaining
8+
9+
$dirs='C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Tools\Llvm',
10+
'C:\rtools45','C:\ghcup','C:\Program Files (x86)\Android',
11+
'C:\Program Files\Google\Chrome','C:\Program Files (x86)\Microsoft\Edge',
12+
'C:\Program Files\Mozilla Firefox','C:\Program Files\MySQL','C:\Julia',
13+
'C:\Program Files\MongoDB','C:\Program Files\Azure Cosmos DB Emulator',
14+
'C:\Program Files\PostgreSQL','C:\Program Files\Unity Hub',
15+
'C:\Strawberry','C:\hostedtoolcache\windows\Java_Temurin-Hotspot_jdk'
16+
17+
foreach ($dirin$dirs) {
18+
Start-ThreadJob-InputObject $dir {
19+
Remove-Item-Recurse -Force -LiteralPath $input
20+
} |Out-Null
21+
}
22+
23+
foreach ($jobinGet-Job) {
24+
Wait-Job$job|Out-Null
25+
if ($job.Error) {
26+
Write-Output"::warning file=$PSCommandPath::$($job.Error)"
27+
}
28+
Remove-Job$job
29+
}
30+
31+
Get-Volume|Out-String|Write-Output
32+
33+
$saved= ($(Get-Volume C).SizeRemaining -$available) /1gb
34+
$savedRounded= [math]::Round($saved,3)
35+
Write-Output"total space saved: $savedRounded GB"

0 commit comments

Comments
 (0)