Add base Docker image for Linux CI jobs

.ci/docker/README.md

@@ -0,0 +1,23 @@
+# Docker images for Executorch CI
+
+This directory contains everything needed to build the Docker images
+that are used in Executorch CI.
+
+## Contents
+
+* `build.sh` -- dispatch script to launch all builds
+* `common` -- scripts used to execute individual Docker build stages
+* `ubuntu` -- Dockerfile for Ubuntu image for CPU build and test jobs
+
+## Usage
+
+```bash
+# Generic usage
+./build.sh "${IMAGE_NAME}" "${DOCKER_BUILD_PARAMETERS}"
+
+# Build a specific image
+./build.sh executorch-ubuntu-22.04-clang12 -t myimage:latest
+
+# Set CLANG version (see build.sh) and build image
+CLANG_VERSION=11 ./build.sh executorch-ubuntu-22.04-clang11 -t myimage:latest
+```

.ci/docker/build.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -ex
+
+IMAGE_NAME="$1"
+shift
+
+echo "Building ${IMAGE_NAME} Docker image"
+
+OS="ubuntu"
+OS_VERSION=22.04
+CLANG_VERSION=12
+
+docker build \
+ --no-cache \
+ --progress=plain \
+ --build-arg "OS_VERSION=${OS_VERSION}" \
+ --build-arg "CLANG_VERSION=${CLANG_VERSION}" \
+ -f "${OS}"/Dockerfile \
+ "$@" \
+ .

.ci/docker/common/install_base.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+
+ apt-get install -y --no-install-recommends \
+ build-essential \
+ ca-certificates \
+ curl \
+ git \
+ wget \
+ sudo \
+ vim \
+ jq \
+ vim \
+ unzip \
+ gdb
+
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+esac

.ci/docker/common/install_buck.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+ apt-get install -y zstd
+
+ wget -q https://github.com/facebook/buck2/releases/download/2023-07-18/buck2-x86_64-unknown-linux-gnu.zst
+ zstd -d buck2-x86_64-unknown-linux-gnu.zst -o buck2
+
+ chmod +x buck2
+ mv buck2 /usr/bin/
+
+ rm buck2-x86_64-unknown-linux-gnu.zst
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+esac

.ci/docker/common/install_clang.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+
+ apt-get install -y --no-install-recommends clang-"$CLANG_VERSION"
+ apt-get install -y --no-install-recommends llvm-"$CLANG_VERSION"
+
+ # Use update-alternatives to make this version the default
+ update-alternatives --install /usr/bin/clang clang /usr/bin/clang-"$CLANG_VERSION" 50
+ update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-"$CLANG_VERSION" 50
+ # Override cc/c++ to clang as well
+ update-alternatives --install /usr/bin/cc cc /usr/bin/clang 50
+ update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 50
+
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+if [ -n "$CLANG_VERSION" ]; then
+ # Install base packages depending on the base OS
+ ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+ case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+ esac
+fi

.ci/docker/common/install_user.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -ex
+
+# Same as ec2-user
+echo "ci-user:x:1000:1000::/var/lib/ci-user:" >> /etc/passwd
+echo "ci-user:x:1000:" >> /etc/group
+# Needed on Focal or newer
+echo "ci-user:*:19110:0:99999:7:::" >> /etc/shadow
+
+# Create $HOME
+mkdir -p /var/lib/ci-user
+chown ci-user:ci-user /var/lib/ci-user
+
+# Allow sudo
+echo 'ci-user ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/ci-user
+
+# Test that sudo works
+sudo -u ci-user sudo -v

.ci/docker/ubuntu/Dockerfile

@@ -0,0 +1,27 @@
+ARG OS_VERSION
+
+FROM ubuntu:${OS_VERSION}
+
+ARG OS_VERSION
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install common dependencies
+COPY ./common/install_base.sh install_base.sh
+RUN bash ./install_base.sh && rm install_base.sh
+
+# Install clang
+ARG CLANG_VERSION
+COPY ./common/install_clang.sh install_clang.sh
+RUN bash ./install_clang.sh && rm install_clang.sh
+
+# Setup buck
+COPY ./common/install_buck.sh install_buck.sh
+RUN bash ./install_buck.sh && rm install_buck.sh
+
+# Setup user
+COPY ./common/install_user.sh install_user.sh
+RUN bash ./install_user.sh && rm install_user.sh
+
+USER ci-user
+CMD ["bash"]

.github/workflows/docker-builds.yml

@@ -0,0 +1,59 @@
+name: docker-builds
+
+on:
+ workflow_dispatch:
+ pull_request:
+ paths:
+ - .ci/docker/**
+ - .github/workflows/docker-builds.yml
+ push:
+ branches:
+ - main
+ schedule:
+ - cron: 1 3 * * 3
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
+ cancel-in-progress: true
+
+jobs:
+ docker-build:
+ runs-on: [self-hosted, linux.2xlarge]
+ timeout-minutes: 240
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - docker-image-name: executorch-ubuntu-22.04-clang12
+ env:
+ DOCKER_IMAGE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ matrix.docker-image-name }}
+ steps:
+ - name: Clean workspace
+ shell: bash
+ run: |
+ echo "${GITHUB_WORKSPACE}"
+ sudo rm -rf "${GITHUB_WORKSPACE}"
+ mkdir "${GITHUB_WORKSPACE}"
+
+ - name: Setup SSH (Click me for login details)
+ uses: pytorch/test-infra/.github/actions/setup-ssh@main
+ with:
+ github-secret: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Checkout Executorch
+ uses: actions/checkout@v3
+
+ - name: Setup Linux
+ uses: pytorch/test-infra/.github/actions/setup-linux@main
+
+ - name: Build docker image
+ id: build-docker-image
+ uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
+ with:
+ docker-image-name: ${{ matrix.docker-image-name }}
+ always-rebuild: true
+ push: true
+
+ - name: Teardown Linux
+ uses: pytorch/test-infra/.github/actions/teardown-linux@main
+ if: always()

.ci/docker/README.md

@@ -0,0 +1,23 @@
+# Docker images for Executorch CI
+
+This directory contains everything needed to build the Docker images
+that are used in Executorch CI.
+
+## Contents
+
+* `build.sh` -- dispatch script to launch all builds
+* `common` -- scripts used to execute individual Docker build stages
+* `ubuntu` -- Dockerfile for Ubuntu image for CPU build and test jobs
+
+## Usage
+
+```bash
+# Generic usage
+./build.sh "${IMAGE_NAME}" "${DOCKER_BUILD_PARAMETERS}"
+
+# Build a specific image
+./build.sh executorch-ubuntu-22.04-clang12 -t myimage:latest
+
+# Set CLANG version (see build.sh) and build image
+CLANG_VERSION=11 ./build.sh executorch-ubuntu-22.04-clang11 -t myimage:latest
+```

.ci/docker/build.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -ex
+
+IMAGE_NAME="$1"
+shift
+
+echo "Building ${IMAGE_NAME} Docker image"
+
+OS="ubuntu"
+OS_VERSION=22.04
+CLANG_VERSION=12
+
+docker build \
+ --no-cache \
+ --progress=plain \
+ --build-arg "OS_VERSION=${OS_VERSION}" \
+ --build-arg "CLANG_VERSION=${CLANG_VERSION}" \
+ -f "${OS}"/Dockerfile \
+ "$@" \
+ .

.ci/docker/common/install_base.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+
+ apt-get install -y --no-install-recommends \
+ build-essential \
+ ca-certificates \
+ curl \
+ git \
+ wget \
+ sudo \
+ vim \
+ jq \
+ vim \
+ unzip \
+ gdb
+
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+esac

.ci/docker/common/install_buck.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+ apt-get install -y zstd
+
+ wget -q https://github.com/facebook/buck2/releases/download/2023-07-18/buck2-x86_64-unknown-linux-gnu.zst
+ zstd -d buck2-x86_64-unknown-linux-gnu.zst -o buck2
+
+ chmod +x buck2
+ mv buck2 /usr/bin/
+
+ rm buck2-x86_64-unknown-linux-gnu.zst
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+esac

.ci/docker/common/install_clang.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+
+ apt-get install -y --no-install-recommends clang-"$CLANG_VERSION"
+ apt-get install -y --no-install-recommends llvm-"$CLANG_VERSION"
+
+ # Use update-alternatives to make this version the default
+ update-alternatives --install /usr/bin/clang clang /usr/bin/clang-"$CLANG_VERSION" 50
+ update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-"$CLANG_VERSION" 50
+ # Override cc/c++ to clang as well
+ update-alternatives --install /usr/bin/cc cc /usr/bin/clang 50
+ update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 50
+
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+if [ -n "$CLANG_VERSION" ]; then
+ # Install base packages depending on the base OS
+ ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+ case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+ esac
+fi

.ci/docker/common/install_user.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -ex
+
+# Same as ec2-user
+echo "ci-user:x:1000:1000::/var/lib/ci-user:" >> /etc/passwd
+echo "ci-user:x:1000:" >> /etc/group
+# Needed on Focal or newer
+echo "ci-user:*:19110:0:99999:7:::" >> /etc/shadow
+
+# Create $HOME
+mkdir -p /var/lib/ci-user
+chown ci-user:ci-user /var/lib/ci-user
+
+# Allow sudo
+echo 'ci-user ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/ci-user
+
+# Test that sudo works
+sudo -u ci-user sudo -v

.ci/docker/ubuntu/Dockerfile

@@ -0,0 +1,27 @@
+ARG OS_VERSION
+
+FROM ubuntu:${OS_VERSION}
+
+ARG OS_VERSION
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install common dependencies
+COPY ./common/install_base.sh install_base.sh
+RUN bash ./install_base.sh && rm install_base.sh
+
+# Install clang
+ARG CLANG_VERSION
+COPY ./common/install_clang.sh install_clang.sh
+RUN bash ./install_clang.sh && rm install_clang.sh
+
+# Setup buck
+COPY ./common/install_buck.sh install_buck.sh
+RUN bash ./install_buck.sh && rm install_buck.sh
+
+# Setup user
+COPY ./common/install_user.sh install_user.sh
+RUN bash ./install_user.sh && rm install_user.sh
+
+USER ci-user
+CMD ["bash"]

.github/workflows/docker-builds.yml

@@ -0,0 +1,59 @@
+name: docker-builds
+
+on:
+ workflow_dispatch:
+ pull_request:
+ paths:
+ - .ci/docker/**
+ - .github/workflows/docker-builds.yml
+ push:
+ branches:
+ - main
+ schedule:
+ - cron: 1 3 * * 3
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
+ cancel-in-progress: true
+
+jobs:
+ docker-build:
+ runs-on: [self-hosted, linux.2xlarge]
+ timeout-minutes: 240
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - docker-image-name: executorch-ubuntu-22.04-clang12
+ env:
+ DOCKER_IMAGE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ matrix.docker-image-name }}
+ steps:
+ - name: Clean workspace
+ shell: bash
+ run: |
+ echo "${GITHUB_WORKSPACE}"
+ sudo rm -rf "${GITHUB_WORKSPACE}"
+ mkdir "${GITHUB_WORKSPACE}"
+
+ - name: Setup SSH (Click me for login details)
+ uses: pytorch/test-infra/.github/actions/setup-ssh@main
+ with:
+ github-secret: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Checkout Executorch
+ uses: actions/checkout@v3
+
+ - name: Setup Linux
+ uses: pytorch/test-infra/.github/actions/setup-linux@main
+
+ - name: Build docker image
+ id: build-docker-image
+ uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
+ with:
+ docker-image-name: ${{ matrix.docker-image-name }}
+ always-rebuild: true
+ push: true
+
+ - name: Teardown Linux
+ uses: pytorch/test-infra/.github/actions/teardown-linux@main
+ if: always()

.ci/docker/README.md

@@ -0,0 +1,23 @@
+# Docker images for Executorch CI
+
+This directory contains everything needed to build the Docker images
+that are used in Executorch CI.
+
+## Contents
+
+* `build.sh` -- dispatch script to launch all builds
+* `common` -- scripts used to execute individual Docker build stages
+* `ubuntu` -- Dockerfile for Ubuntu image for CPU build and test jobs
+
+## Usage
+
+```bash
+# Generic usage
+./build.sh "${IMAGE_NAME}" "${DOCKER_BUILD_PARAMETERS}"
+
+# Build a specific image
+./build.sh executorch-ubuntu-22.04-clang12 -t myimage:latest
+
+# Set CLANG version (see build.sh) and build image
+CLANG_VERSION=11 ./build.sh executorch-ubuntu-22.04-clang11 -t myimage:latest
+```

.ci/docker/build.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -ex
+
+IMAGE_NAME="$1"
+shift
+
+echo "Building ${IMAGE_NAME} Docker image"
+
+OS="ubuntu"
+OS_VERSION=22.04
+CLANG_VERSION=12
+
+docker build \
+ --no-cache \
+ --progress=plain \
+ --build-arg "OS_VERSION=${OS_VERSION}" \
+ --build-arg "CLANG_VERSION=${CLANG_VERSION}" \
+ -f "${OS}"/Dockerfile \
+ "$@" \
+ .

.ci/docker/common/install_base.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+
+ apt-get install -y --no-install-recommends \
+ build-essential \
+ ca-certificates \
+ curl \
+ git \
+ wget \
+ sudo \
+ vim \
+ jq \
+ vim \
+ unzip \
+ gdb
+
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+esac

.ci/docker/common/install_buck.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+ apt-get install -y zstd
+
+ wget -q https://github.com/facebook/buck2/releases/download/2023-07-18/buck2-x86_64-unknown-linux-gnu.zst
+ zstd -d buck2-x86_64-unknown-linux-gnu.zst -o buck2
+
+ chmod +x buck2
+ mv buck2 /usr/bin/
+
+ rm buck2-x86_64-unknown-linux-gnu.zst
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+esac

.ci/docker/common/install_clang.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+
+ apt-get install -y --no-install-recommends clang-"$CLANG_VERSION"
+ apt-get install -y --no-install-recommends llvm-"$CLANG_VERSION"
+
+ # Use update-alternatives to make this version the default
+ update-alternatives --install /usr/bin/clang clang /usr/bin/clang-"$CLANG_VERSION" 50
+ update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-"$CLANG_VERSION" 50
+ # Override cc/c++ to clang as well
+ update-alternatives --install /usr/bin/cc cc /usr/bin/clang 50
+ update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 50
+
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+if [ -n "$CLANG_VERSION" ]; then
+ # Install base packages depending on the base OS
+ ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+ case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+ esac
+fi

.ci/docker/common/install_user.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -ex
+
+# Same as ec2-user
+echo "ci-user:x:1000:1000::/var/lib/ci-user:" >> /etc/passwd
+echo "ci-user:x:1000:" >> /etc/group
+# Needed on Focal or newer
+echo "ci-user:*:19110:0:99999:7:::" >> /etc/shadow
+
+# Create $HOME
+mkdir -p /var/lib/ci-user
+chown ci-user:ci-user /var/lib/ci-user
+
+# Allow sudo
+echo 'ci-user ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/ci-user
+
+# Test that sudo works
+sudo -u ci-user sudo -v

.ci/docker/ubuntu/Dockerfile

@@ -0,0 +1,27 @@
+ARG OS_VERSION
+
+FROM ubuntu:${OS_VERSION}
+
+ARG OS_VERSION
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install common dependencies
+COPY ./common/install_base.sh install_base.sh
+RUN bash ./install_base.sh && rm install_base.sh
+
+# Install clang
+ARG CLANG_VERSION
+COPY ./common/install_clang.sh install_clang.sh
+RUN bash ./install_clang.sh && rm install_clang.sh
+
+# Setup buck
+COPY ./common/install_buck.sh install_buck.sh
+RUN bash ./install_buck.sh && rm install_buck.sh
+
+# Setup user
+COPY ./common/install_user.sh install_user.sh
+RUN bash ./install_user.sh && rm install_user.sh
+
+USER ci-user
+CMD ["bash"]

.github/workflows/docker-builds.yml

@@ -0,0 +1,59 @@
+name: docker-builds
+
+on:
+ workflow_dispatch:
+ pull_request:
+ paths:
+ - .ci/docker/**
+ - .github/workflows/docker-builds.yml
+ push:
+ branches:
+ - main
+ schedule:
+ - cron: 1 3 * * 3
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
+ cancel-in-progress: true
+
+jobs:
+ docker-build:
+ runs-on: [self-hosted, linux.2xlarge]
+ timeout-minutes: 240
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - docker-image-name: executorch-ubuntu-22.04-clang12
+ env:
+ DOCKER_IMAGE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ matrix.docker-image-name }}
+ steps:
+ - name: Clean workspace
+ shell: bash
+ run: |
+ echo "${GITHUB_WORKSPACE}"
+ sudo rm -rf "${GITHUB_WORKSPACE}"
+ mkdir "${GITHUB_WORKSPACE}"
+
+ - name: Setup SSH (Click me for login details)
+ uses: pytorch/test-infra/.github/actions/setup-ssh@main
+ with:
+ github-secret: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Checkout Executorch
+ uses: actions/checkout@v3
+
+ - name: Setup Linux
+ uses: pytorch/test-infra/.github/actions/setup-linux@main
+
+ - name: Build docker image
+ id: build-docker-image
+ uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
+ with:
+ docker-image-name: ${{ matrix.docker-image-name }}
+ always-rebuild: true
+ push: true
+
+ - name: Teardown Linux
+ uses: pytorch/test-infra/.github/actions/teardown-linux@main
+ if: always()

.ci/docker/README.md

@@ -0,0 +1,23 @@
+# Docker images for Executorch CI
+
+This directory contains everything needed to build the Docker images
+that are used in Executorch CI.
+
+## Contents
+
+* `build.sh` -- dispatch script to launch all builds
+* `common` -- scripts used to execute individual Docker build stages
+* `ubuntu` -- Dockerfile for Ubuntu image for CPU build and test jobs
+
+## Usage
+
+```bash
+# Generic usage
+./build.sh "${IMAGE_NAME}" "${DOCKER_BUILD_PARAMETERS}"
+
+# Build a specific image
+./build.sh executorch-ubuntu-22.04-clang12 -t myimage:latest
+
+# Set CLANG version (see build.sh) and build image
+CLANG_VERSION=11 ./build.sh executorch-ubuntu-22.04-clang11 -t myimage:latest
+```

.ci/docker/build.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -ex
+
+IMAGE_NAME="$1"
+shift
+
+echo "Building ${IMAGE_NAME} Docker image"
+
+OS="ubuntu"
+OS_VERSION=22.04
+CLANG_VERSION=12
+
+docker build \
+ --no-cache \
+ --progress=plain \
+ --build-arg "OS_VERSION=${OS_VERSION}" \
+ --build-arg "CLANG_VERSION=${CLANG_VERSION}" \
+ -f "${OS}"/Dockerfile \
+ "$@" \
+ .

.ci/docker/common/install_base.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+
+ apt-get install -y --no-install-recommends \
+ build-essential \
+ ca-certificates \
+ curl \
+ git \
+ wget \
+ sudo \
+ vim \
+ jq \
+ vim \
+ unzip \
+ gdb
+
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+esac

.ci/docker/common/install_buck.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+ apt-get install -y zstd
+
+ wget -q https://github.com/facebook/buck2/releases/download/2023-07-18/buck2-x86_64-unknown-linux-gnu.zst
+ zstd -d buck2-x86_64-unknown-linux-gnu.zst -o buck2
+
+ chmod +x buck2
+ mv buck2 /usr/bin/
+
+ rm buck2-x86_64-unknown-linux-gnu.zst
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+esac

.ci/docker/common/install_clang.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+
+ apt-get install -y --no-install-recommends clang-"$CLANG_VERSION"
+ apt-get install -y --no-install-recommends llvm-"$CLANG_VERSION"
+
+ # Use update-alternatives to make this version the default
+ update-alternatives --install /usr/bin/clang clang /usr/bin/clang-"$CLANG_VERSION" 50
+ update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-"$CLANG_VERSION" 50
+ # Override cc/c++ to clang as well
+ update-alternatives --install /usr/bin/cc cc /usr/bin/clang 50
+ update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 50
+
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+if [ -n "$CLANG_VERSION" ]; then
+ # Install base packages depending on the base OS
+ ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+ case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+ esac
+fi

.ci/docker/common/install_user.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -ex
+
+# Same as ec2-user
+echo "ci-user:x:1000:1000::/var/lib/ci-user:" >> /etc/passwd
+echo "ci-user:x:1000:" >> /etc/group
+# Needed on Focal or newer
+echo "ci-user:*:19110:0:99999:7:::" >> /etc/shadow
+
+# Create $HOME
+mkdir -p /var/lib/ci-user
+chown ci-user:ci-user /var/lib/ci-user
+
+# Allow sudo
+echo 'ci-user ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/ci-user
+
+# Test that sudo works
+sudo -u ci-user sudo -v

.ci/docker/ubuntu/Dockerfile

@@ -0,0 +1,27 @@
+ARG OS_VERSION
+
+FROM ubuntu:${OS_VERSION}
+
+ARG OS_VERSION
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install common dependencies
+COPY ./common/install_base.sh install_base.sh
+RUN bash ./install_base.sh && rm install_base.sh
+
+# Install clang
+ARG CLANG_VERSION
+COPY ./common/install_clang.sh install_clang.sh
+RUN bash ./install_clang.sh && rm install_clang.sh
+
+# Setup buck
+COPY ./common/install_buck.sh install_buck.sh
+RUN bash ./install_buck.sh && rm install_buck.sh
+
+# Setup user
+COPY ./common/install_user.sh install_user.sh
+RUN bash ./install_user.sh && rm install_user.sh
+
+USER ci-user
+CMD ["bash"]

.github/workflows/docker-builds.yml

@@ -0,0 +1,59 @@
+name: docker-builds
+
+on:
+ workflow_dispatch:
+ pull_request:
+ paths:
+ - .ci/docker/**
+ - .github/workflows/docker-builds.yml
+ push:
+ branches:
+ - main
+ schedule:
+ - cron: 1 3 * * 3
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
+ cancel-in-progress: true
+
+jobs:
+ docker-build:
+ runs-on: [self-hosted, linux.2xlarge]
+ timeout-minutes: 240
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - docker-image-name: executorch-ubuntu-22.04-clang12
+ env:
+ DOCKER_IMAGE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ matrix.docker-image-name }}
+ steps:
+ - name: Clean workspace
+ shell: bash
+ run: |
+ echo "${GITHUB_WORKSPACE}"
+ sudo rm -rf "${GITHUB_WORKSPACE}"
+ mkdir "${GITHUB_WORKSPACE}"
+
+ - name: Setup SSH (Click me for login details)
+ uses: pytorch/test-infra/.github/actions/setup-ssh@main
+ with:
+ github-secret: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Checkout Executorch
+ uses: actions/checkout@v3
+
+ - name: Setup Linux
+ uses: pytorch/test-infra/.github/actions/setup-linux@main
+
+ - name: Build docker image
+ id: build-docker-image
+ uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
+ with:
+ docker-image-name: ${{ matrix.docker-image-name }}
+ always-rebuild: true
+ push: true
+
+ - name: Teardown Linux
+ uses: pytorch/test-infra/.github/actions/teardown-linux@main
+ if: always()

.ci/docker/README.md

@@ -0,0 +1,23 @@
+# Docker images for Executorch CI
+
+This directory contains everything needed to build the Docker images
+that are used in Executorch CI.
+
+## Contents
+
+* `build.sh` -- dispatch script to launch all builds
+* `common` -- scripts used to execute individual Docker build stages
+* `ubuntu` -- Dockerfile for Ubuntu image for CPU build and test jobs
+
+## Usage
+
+```bash
+# Generic usage
+./build.sh "${IMAGE_NAME}" "${DOCKER_BUILD_PARAMETERS}"
+
+# Build a specific image
+./build.sh executorch-ubuntu-22.04-clang12 -t myimage:latest
+
+# Set CLANG version (see build.sh) and build image
+CLANG_VERSION=11 ./build.sh executorch-ubuntu-22.04-clang11 -t myimage:latest
+```

.ci/docker/build.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -ex
+
+IMAGE_NAME="$1"
+shift
+
+echo "Building ${IMAGE_NAME} Docker image"
+
+OS="ubuntu"
+OS_VERSION=22.04
+CLANG_VERSION=12
+
+docker build \
+ --no-cache \
+ --progress=plain \
+ --build-arg "OS_VERSION=${OS_VERSION}" \
+ --build-arg "CLANG_VERSION=${CLANG_VERSION}" \
+ -f "${OS}"/Dockerfile \
+ "$@" \
+ .

.ci/docker/common/install_base.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+
+ apt-get install -y --no-install-recommends \
+ build-essential \
+ ca-certificates \
+ curl \
+ git \
+ wget \
+ sudo \
+ vim \
+ jq \
+ vim \
+ unzip \
+ gdb
+
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+esac

.ci/docker/common/install_buck.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+ apt-get install -y zstd
+
+ wget -q https://github.com/facebook/buck2/releases/download/2023-07-18/buck2-x86_64-unknown-linux-gnu.zst
+ zstd -d buck2-x86_64-unknown-linux-gnu.zst -o buck2
+
+ chmod +x buck2
+ mv buck2 /usr/bin/
+
+ rm buck2-x86_64-unknown-linux-gnu.zst
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+esac

.ci/docker/common/install_clang.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+
+ apt-get install -y --no-install-recommends clang-"$CLANG_VERSION"
+ apt-get install -y --no-install-recommends llvm-"$CLANG_VERSION"
+
+ # Use update-alternatives to make this version the default
+ update-alternatives --install /usr/bin/clang clang /usr/bin/clang-"$CLANG_VERSION" 50
+ update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-"$CLANG_VERSION" 50
+ # Override cc/c++ to clang as well
+ update-alternatives --install /usr/bin/cc cc /usr/bin/clang 50
+ update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 50
+
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+if [ -n "$CLANG_VERSION" ]; then
+ # Install base packages depending on the base OS
+ ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+ case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+ esac
+fi

.ci/docker/common/install_user.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -ex
+
+# Same as ec2-user
+echo "ci-user:x:1000:1000::/var/lib/ci-user:" >> /etc/passwd
+echo "ci-user:x:1000:" >> /etc/group
+# Needed on Focal or newer
+echo "ci-user:*:19110:0:99999:7:::" >> /etc/shadow
+
+# Create $HOME
+mkdir -p /var/lib/ci-user
+chown ci-user:ci-user /var/lib/ci-user
+
+# Allow sudo
+echo 'ci-user ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/ci-user
+
+# Test that sudo works
+sudo -u ci-user sudo -v

.ci/docker/ubuntu/Dockerfile

@@ -0,0 +1,27 @@
+ARG OS_VERSION
+
+FROM ubuntu:${OS_VERSION}
+
+ARG OS_VERSION
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install common dependencies
+COPY ./common/install_base.sh install_base.sh
+RUN bash ./install_base.sh && rm install_base.sh
+
+# Install clang
+ARG CLANG_VERSION
+COPY ./common/install_clang.sh install_clang.sh
+RUN bash ./install_clang.sh && rm install_clang.sh
+
+# Setup buck
+COPY ./common/install_buck.sh install_buck.sh
+RUN bash ./install_buck.sh && rm install_buck.sh
+
+# Setup user
+COPY ./common/install_user.sh install_user.sh
+RUN bash ./install_user.sh && rm install_user.sh
+
+USER ci-user
+CMD ["bash"]

.github/workflows/docker-builds.yml

@@ -0,0 +1,59 @@
+name: docker-builds
+
+on:
+ workflow_dispatch:
+ pull_request:
+ paths:
+ - .ci/docker/**
+ - .github/workflows/docker-builds.yml
+ push:
+ branches:
+ - main
+ schedule:
+ - cron: 1 3 * * 3
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
+ cancel-in-progress: true
+
+jobs:
+ docker-build:
+ runs-on: [self-hosted, linux.2xlarge]
+ timeout-minutes: 240
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - docker-image-name: executorch-ubuntu-22.04-clang12
+ env:
+ DOCKER_IMAGE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ matrix.docker-image-name }}
+ steps:
+ - name: Clean workspace
+ shell: bash
+ run: |
+ echo "${GITHUB_WORKSPACE}"
+ sudo rm -rf "${GITHUB_WORKSPACE}"
+ mkdir "${GITHUB_WORKSPACE}"
+
+ - name: Setup SSH (Click me for login details)
+ uses: pytorch/test-infra/.github/actions/setup-ssh@main
+ with:
+ github-secret: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Checkout Executorch
+ uses: actions/checkout@v3
+
+ - name: Setup Linux
+ uses: pytorch/test-infra/.github/actions/setup-linux@main
+
+ - name: Build docker image
+ id: build-docker-image
+ uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
+ with:
+ docker-image-name: ${{ matrix.docker-image-name }}
+ always-rebuild: true
+ push: true
+
+ - name: Teardown Linux
+ uses: pytorch/test-infra/.github/actions/teardown-linux@main
+ if: always()

.ci/docker/README.md

@@ -0,0 +1,23 @@
+# Docker images for Executorch CI
+
+This directory contains everything needed to build the Docker images
+that are used in Executorch CI.
+
+## Contents
+
+* `build.sh` -- dispatch script to launch all builds
+* `common` -- scripts used to execute individual Docker build stages
+* `ubuntu` -- Dockerfile for Ubuntu image for CPU build and test jobs
+
+## Usage
+
+```bash
+# Generic usage
+./build.sh "${IMAGE_NAME}" "${DOCKER_BUILD_PARAMETERS}"
+
+# Build a specific image
+./build.sh executorch-ubuntu-22.04-clang12 -t myimage:latest
+
+# Set CLANG version (see build.sh) and build image
+CLANG_VERSION=11 ./build.sh executorch-ubuntu-22.04-clang11 -t myimage:latest
+```

.ci/docker/build.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -ex
+
+IMAGE_NAME="$1"
+shift
+
+echo "Building ${IMAGE_NAME} Docker image"
+
+OS="ubuntu"
+OS_VERSION=22.04
+CLANG_VERSION=12
+
+docker build \
+ --no-cache \
+ --progress=plain \
+ --build-arg "OS_VERSION=${OS_VERSION}" \
+ --build-arg "CLANG_VERSION=${CLANG_VERSION}" \
+ -f "${OS}"/Dockerfile \
+ "$@" \
+ .

.ci/docker/common/install_base.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+
+ apt-get install -y --no-install-recommends \
+ build-essential \
+ ca-certificates \
+ curl \
+ git \
+ wget \
+ sudo \
+ vim \
+ jq \
+ vim \
+ unzip \
+ gdb
+
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+esac

.ci/docker/common/install_buck.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+ apt-get install -y zstd
+
+ wget -q https://github.com/facebook/buck2/releases/download/2023-07-18/buck2-x86_64-unknown-linux-gnu.zst
+ zstd -d buck2-x86_64-unknown-linux-gnu.zst -o buck2
+
+ chmod +x buck2
+ mv buck2 /usr/bin/
+
+ rm buck2-x86_64-unknown-linux-gnu.zst
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+esac

.ci/docker/common/install_clang.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+
+ apt-get install -y --no-install-recommends clang-"$CLANG_VERSION"
+ apt-get install -y --no-install-recommends llvm-"$CLANG_VERSION"
+
+ # Use update-alternatives to make this version the default
+ update-alternatives --install /usr/bin/clang clang /usr/bin/clang-"$CLANG_VERSION" 50
+ update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-"$CLANG_VERSION" 50
+ # Override cc/c++ to clang as well
+ update-alternatives --install /usr/bin/cc cc /usr/bin/clang 50
+ update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 50
+
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+if [ -n "$CLANG_VERSION" ]; then
+ # Install base packages depending on the base OS
+ ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+ case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+ esac
+fi

.ci/docker/common/install_user.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -ex
+
+# Same as ec2-user
+echo "ci-user:x:1000:1000::/var/lib/ci-user:" >> /etc/passwd
+echo "ci-user:x:1000:" >> /etc/group
+# Needed on Focal or newer
+echo "ci-user:*:19110:0:99999:7:::" >> /etc/shadow
+
+# Create $HOME
+mkdir -p /var/lib/ci-user
+chown ci-user:ci-user /var/lib/ci-user
+
+# Allow sudo
+echo 'ci-user ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/ci-user
+
+# Test that sudo works
+sudo -u ci-user sudo -v

.ci/docker/ubuntu/Dockerfile

@@ -0,0 +1,27 @@
+ARG OS_VERSION
+
+FROM ubuntu:${OS_VERSION}
+
+ARG OS_VERSION
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install common dependencies
+COPY ./common/install_base.sh install_base.sh
+RUN bash ./install_base.sh && rm install_base.sh
+
+# Install clang
+ARG CLANG_VERSION
+COPY ./common/install_clang.sh install_clang.sh
+RUN bash ./install_clang.sh && rm install_clang.sh
+
+# Setup buck
+COPY ./common/install_buck.sh install_buck.sh
+RUN bash ./install_buck.sh && rm install_buck.sh
+
+# Setup user
+COPY ./common/install_user.sh install_user.sh
+RUN bash ./install_user.sh && rm install_user.sh
+
+USER ci-user
+CMD ["bash"]

.github/workflows/docker-builds.yml

@@ -0,0 +1,59 @@
+name: docker-builds
+
+on:
+ workflow_dispatch:
+ pull_request:
+ paths:
+ - .ci/docker/**
+ - .github/workflows/docker-builds.yml
+ push:
+ branches:
+ - main
+ schedule:
+ - cron: 1 3 * * 3
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
+ cancel-in-progress: true
+
+jobs:
+ docker-build:
+ runs-on: [self-hosted, linux.2xlarge]
+ timeout-minutes: 240
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - docker-image-name: executorch-ubuntu-22.04-clang12
+ env:
+ DOCKER_IMAGE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ matrix.docker-image-name }}
+ steps:
+ - name: Clean workspace
+ shell: bash
+ run: |
+ echo "${GITHUB_WORKSPACE}"
+ sudo rm -rf "${GITHUB_WORKSPACE}"
+ mkdir "${GITHUB_WORKSPACE}"
+
+ - name: Setup SSH (Click me for login details)
+ uses: pytorch/test-infra/.github/actions/setup-ssh@main
+ with:
+ github-secret: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Checkout Executorch
+ uses: actions/checkout@v3
+
+ - name: Setup Linux
+ uses: pytorch/test-infra/.github/actions/setup-linux@main
+
+ - name: Build docker image
+ id: build-docker-image
+ uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
+ with:
+ docker-image-name: ${{ matrix.docker-image-name }}
+ always-rebuild: true
+ push: true
+
+ - name: Teardown Linux
+ uses: pytorch/test-infra/.github/actions/teardown-linux@main
+ if: always()

.ci/docker/README.md

@@ -0,0 +1,23 @@
+# Docker images for Executorch CI
+
+This directory contains everything needed to build the Docker images
+that are used in Executorch CI.
+
+## Contents
+
+* `build.sh` -- dispatch script to launch all builds
+* `common` -- scripts used to execute individual Docker build stages
+* `ubuntu` -- Dockerfile for Ubuntu image for CPU build and test jobs
+
+## Usage
+
+```bash
+# Generic usage
+./build.sh "${IMAGE_NAME}" "${DOCKER_BUILD_PARAMETERS}"
+
+# Build a specific image
+./build.sh executorch-ubuntu-22.04-clang12 -t myimage:latest
+
+# Set CLANG version (see build.sh) and build image
+CLANG_VERSION=11 ./build.sh executorch-ubuntu-22.04-clang11 -t myimage:latest
+```

.ci/docker/build.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -ex
+
+IMAGE_NAME="$1"
+shift
+
+echo "Building ${IMAGE_NAME} Docker image"
+
+OS="ubuntu"
+OS_VERSION=22.04
+CLANG_VERSION=12
+
+docker build \
+ --no-cache \
+ --progress=plain \
+ --build-arg "OS_VERSION=${OS_VERSION}" \
+ --build-arg "CLANG_VERSION=${CLANG_VERSION}" \
+ -f "${OS}"/Dockerfile \
+ "$@" \
+ .

.ci/docker/common/install_base.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+
+ apt-get install -y --no-install-recommends \
+ build-essential \
+ ca-certificates \
+ curl \
+ git \
+ wget \
+ sudo \
+ vim \
+ jq \
+ vim \
+ unzip \
+ gdb
+
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+esac

.ci/docker/common/install_buck.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+ apt-get install -y zstd
+
+ wget -q https://github.com/facebook/buck2/releases/download/2023-07-18/buck2-x86_64-unknown-linux-gnu.zst
+ zstd -d buck2-x86_64-unknown-linux-gnu.zst -o buck2
+
+ chmod +x buck2
+ mv buck2 /usr/bin/
+
+ rm buck2-x86_64-unknown-linux-gnu.zst
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+esac

.ci/docker/common/install_clang.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+
+ apt-get install -y --no-install-recommends clang-"$CLANG_VERSION"
+ apt-get install -y --no-install-recommends llvm-"$CLANG_VERSION"
+
+ # Use update-alternatives to make this version the default
+ update-alternatives --install /usr/bin/clang clang /usr/bin/clang-"$CLANG_VERSION" 50
+ update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-"$CLANG_VERSION" 50
+ # Override cc/c++ to clang as well
+ update-alternatives --install /usr/bin/cc cc /usr/bin/clang 50
+ update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 50
+
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+if [ -n "$CLANG_VERSION" ]; then
+ # Install base packages depending on the base OS
+ ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+ case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+ esac
+fi

.ci/docker/common/install_user.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -ex
+
+# Same as ec2-user
+echo "ci-user:x:1000:1000::/var/lib/ci-user:" >> /etc/passwd
+echo "ci-user:x:1000:" >> /etc/group
+# Needed on Focal or newer
+echo "ci-user:*:19110:0:99999:7:::" >> /etc/shadow
+
+# Create $HOME
+mkdir -p /var/lib/ci-user
+chown ci-user:ci-user /var/lib/ci-user
+
+# Allow sudo
+echo 'ci-user ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/ci-user
+
+# Test that sudo works
+sudo -u ci-user sudo -v

.ci/docker/ubuntu/Dockerfile

@@ -0,0 +1,27 @@
+ARG OS_VERSION
+
+FROM ubuntu:${OS_VERSION}
+
+ARG OS_VERSION
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install common dependencies
+COPY ./common/install_base.sh install_base.sh
+RUN bash ./install_base.sh && rm install_base.sh
+
+# Install clang
+ARG CLANG_VERSION
+COPY ./common/install_clang.sh install_clang.sh
+RUN bash ./install_clang.sh && rm install_clang.sh
+
+# Setup buck
+COPY ./common/install_buck.sh install_buck.sh
+RUN bash ./install_buck.sh && rm install_buck.sh
+
+# Setup user
+COPY ./common/install_user.sh install_user.sh
+RUN bash ./install_user.sh && rm install_user.sh
+
+USER ci-user
+CMD ["bash"]

.github/workflows/docker-builds.yml

@@ -0,0 +1,59 @@
+name: docker-builds
+
+on:
+ workflow_dispatch:
+ pull_request:
+ paths:
+ - .ci/docker/**
+ - .github/workflows/docker-builds.yml
+ push:
+ branches:
+ - main
+ schedule:
+ - cron: 1 3 * * 3
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
+ cancel-in-progress: true
+
+jobs:
+ docker-build:
+ runs-on: [self-hosted, linux.2xlarge]
+ timeout-minutes: 240
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - docker-image-name: executorch-ubuntu-22.04-clang12
+ env:
+ DOCKER_IMAGE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ matrix.docker-image-name }}
+ steps:
+ - name: Clean workspace
+ shell: bash
+ run: |
+ echo "${GITHUB_WORKSPACE}"
+ sudo rm -rf "${GITHUB_WORKSPACE}"
+ mkdir "${GITHUB_WORKSPACE}"
+
+ - name: Setup SSH (Click me for login details)
+ uses: pytorch/test-infra/.github/actions/setup-ssh@main
+ with:
+ github-secret: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Checkout Executorch
+ uses: actions/checkout@v3
+
+ - name: Setup Linux
+ uses: pytorch/test-infra/.github/actions/setup-linux@main
+
+ - name: Build docker image
+ id: build-docker-image
+ uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
+ with:
+ docker-image-name: ${{ matrix.docker-image-name }}
+ always-rebuild: true
+ push: true
+
+ - name: Teardown Linux
+ uses: pytorch/test-infra/.github/actions/teardown-linux@main
+ if: always()

.ci/docker/README.md

@@ -0,0 +1,23 @@
+# Docker images for Executorch CI
+
+This directory contains everything needed to build the Docker images
+that are used in Executorch CI.
+
+## Contents
+
+* `build.sh` -- dispatch script to launch all builds
+* `common` -- scripts used to execute individual Docker build stages
+* `ubuntu` -- Dockerfile for Ubuntu image for CPU build and test jobs
+
+## Usage
+
+```bash
+# Generic usage
+./build.sh "${IMAGE_NAME}" "${DOCKER_BUILD_PARAMETERS}"
+
+# Build a specific image
+./build.sh executorch-ubuntu-22.04-clang12 -t myimage:latest
+
+# Set CLANG version (see build.sh) and build image
+CLANG_VERSION=11 ./build.sh executorch-ubuntu-22.04-clang11 -t myimage:latest
+```

.ci/docker/build.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -ex
+
+IMAGE_NAME="$1"
+shift
+
+echo "Building ${IMAGE_NAME} Docker image"
+
+OS="ubuntu"
+OS_VERSION=22.04
+CLANG_VERSION=12
+
+docker build \
+ --no-cache \
+ --progress=plain \
+ --build-arg "OS_VERSION=${OS_VERSION}" \
+ --build-arg "CLANG_VERSION=${CLANG_VERSION}" \
+ -f "${OS}"/Dockerfile \
+ "$@" \
+ .

.ci/docker/common/install_base.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+
+ apt-get install -y --no-install-recommends \
+ build-essential \
+ ca-certificates \
+ curl \
+ git \
+ wget \
+ sudo \
+ vim \
+ jq \
+ vim \
+ unzip \
+ gdb
+
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+esac

.ci/docker/common/install_buck.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+ apt-get install -y zstd
+
+ wget -q https://github.com/facebook/buck2/releases/download/2023-07-18/buck2-x86_64-unknown-linux-gnu.zst
+ zstd -d buck2-x86_64-unknown-linux-gnu.zst -o buck2
+
+ chmod +x buck2
+ mv buck2 /usr/bin/
+
+ rm buck2-x86_64-unknown-linux-gnu.zst
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+esac

.ci/docker/common/install_clang.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -ex
+
+install_ubuntu() {
+ apt-get update
+
+ apt-get install -y --no-install-recommends clang-"$CLANG_VERSION"
+ apt-get install -y --no-install-recommends llvm-"$CLANG_VERSION"
+
+ # Use update-alternatives to make this version the default
+ update-alternatives --install /usr/bin/clang clang /usr/bin/clang-"$CLANG_VERSION" 50
+ update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-"$CLANG_VERSION" 50
+ # Override cc/c++ to clang as well
+ update-alternatives --install /usr/bin/cc cc /usr/bin/clang 50
+ update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 50
+
+ # Cleanup package manager
+ apt-get autoclean && apt-get clean
+ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+if [ -n "$CLANG_VERSION" ]; then
+ # Install base packages depending on the base OS
+ ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+ case "$ID" in
+ ubuntu)
+ install_ubuntu
+ ;;
+ *)
+ echo "Unable to determine OS..."
+ exit 1
+ ;;
+ esac
+fi

.ci/docker/common/install_user.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -ex
+
+# Same as ec2-user
+echo "ci-user:x:1000:1000::/var/lib/ci-user:" >> /etc/passwd
+echo "ci-user:x:1000:" >> /etc/group
+# Needed on Focal or newer
+echo "ci-user:*:19110:0:99999:7:::" >> /etc/shadow
+
+# Create $HOME
+mkdir -p /var/lib/ci-user
+chown ci-user:ci-user /var/lib/ci-user
+
+# Allow sudo
+echo 'ci-user ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/ci-user
+
+# Test that sudo works
+sudo -u ci-user sudo -v

.ci/docker/ubuntu/Dockerfile

@@ -0,0 +1,27 @@
+ARG OS_VERSION
+
+FROM ubuntu:${OS_VERSION}
+
+ARG OS_VERSION
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install common dependencies
+COPY ./common/install_base.sh install_base.sh
+RUN bash ./install_base.sh && rm install_base.sh
+
+# Install clang
+ARG CLANG_VERSION
+COPY ./common/install_clang.sh install_clang.sh
+RUN bash ./install_clang.sh && rm install_clang.sh
+
+# Setup buck
+COPY ./common/install_buck.sh install_buck.sh
+RUN bash ./install_buck.sh && rm install_buck.sh
+
+# Setup user
+COPY ./common/install_user.sh install_user.sh
+RUN bash ./install_user.sh && rm install_user.sh
+
+USER ci-user
+CMD ["bash"]

.github/workflows/docker-builds.yml

@@ -0,0 +1,59 @@
+name: docker-builds
+
+on:
+ workflow_dispatch:
+ pull_request:
+ paths:
+ - .ci/docker/**
+ - .github/workflows/docker-builds.yml
+ push:
+ branches:
+ - main
+ schedule:
+ - cron: 1 3 * * 3
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
+ cancel-in-progress: true
+
+jobs:
+ docker-build:
+ runs-on: [self-hosted, linux.2xlarge]
+ timeout-minutes: 240
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - docker-image-name: executorch-ubuntu-22.04-clang12
+ env:
+ DOCKER_IMAGE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ matrix.docker-image-name }}
+ steps:
+ - name: Clean workspace
+ shell: bash
+ run: |
+ echo "${GITHUB_WORKSPACE}"
+ sudo rm -rf "${GITHUB_WORKSPACE}"
+ mkdir "${GITHUB_WORKSPACE}"
+
+ - name: Setup SSH (Click me for login details)
+ uses: pytorch/test-infra/.github/actions/setup-ssh@main
+ with:
+ github-secret: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Checkout Executorch
+ uses: actions/checkout@v3
+
+ - name: Setup Linux
+ uses: pytorch/test-infra/.github/actions/setup-linux@main
+
+ - name: Build docker image
+ id: build-docker-image
+ uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
+ with:
+ docker-image-name: ${{ matrix.docker-image-name }}
+ always-rebuild: true
+ push: true
+
+ - name: Teardown Linux
+ uses: pytorch/test-infra/.github/actions/teardown-linux@main
+ if: always()