From edba1c8e274610563dcaf06117281dc6a0e535d9 Mon Sep 17 00:00:00 2001 From: AJ Slater Date: Mon, 24 Aug 2020 14:15:46 -0700 Subject: [PATCH 01/11] circecli build and push attempt --- .circleci/config.yml | 68 ++++++++++++++++++++--------- Dockerfile | 14 ++---- Dockerfile.build | 32 ++++++++++++-- build-docker-image.sh | 3 +- ci/setup.sh | 30 ------------- docker-compose-build.yaml | 1 + docker-env | 3 ++ docker/usr/local/sbin/entrypoint.sh | 8 ---- docker/usr/local/sbin/moduser.sh | 15 ------- 9 files changed, 85 insertions(+), 89 deletions(-) delete mode 100755 ci/setup.sh delete mode 100755 docker/usr/local/sbin/entrypoint.sh delete mode 100755 docker/usr/local/sbin/moduser.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 3c3bc8763..ec4bee7df 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,36 +1,62 @@ -version: 2.1 +executors: + docker-publisher: + machine: true jobs: - setup-lint-test: - docker: - - image: circleci/python:latest + build: + executor: docker-publisher steps: - checkout - - restore_cache: - keys: - - deps-{{ checksum "pyproject.toml" }} - run: - command: ./ci/setup.sh - name: Setup - - save_cache: - key: deps-{{ checksum "pyproject.toml" }} + command: ./build-wheel.sh + name: Build Python Wheel + - store_test_results: + path: test-results + - run: + command: ./build-docker-image.sh + name: Build Runnable Docker Image + - run: + command: ./ci/archive-image.sh ./<< pipeline.parameters.image_archive >> + name: Archive Docker image + - persist_to_workspace: paths: - - /home/circleci/project/.poetry/virtualenvs + - ./<< pipeline.parameters.image_archive >> + - ./dist + - ./env + - ./ci/tag-and-push.sh + root: . + deploy: + executor: docker-publisher + steps: + - attach_workspace: + at: . - run: - command: ./lint.sh - name: Lint + command: docker load -i "./<< pipeline.parameters.image_archive >>" + name: Load archive Docker image - run: - command: ./test.sh - name: Test - - store_test_results: - path: test-results + command: ./ci/tag-and-push.sh + name: Publish Docker Image to Docker Hub +parameters: + image_archive: + default: image.tar + type: string +version: 2.1 workflows: - version: 2.1 main: jobs: - setup-lint-test: - - filters: + - build: + filters: branches: only: - release - develop - circleci + - deploy: + filters: + branches: + only: + - release + - circleci + require: + - build + version: 2.1 + diff --git a/Dockerfile b/Dockerfile index 89d8c02ac..aeb53977a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM python:alpine AS codex-install +ARG BASE_VERSION +FROM ajslater/python-alpine:$BASE_VERSION AS codex-install RUN echo "**** install system wheel building packages ****" && \ apk add --no-cache \ @@ -17,14 +18,8 @@ RUN pip3 install wheel COPY dist/*.whl /tmp/ RUN pip3 wheel /tmp/*.whl --wheel-dir=/wheels -FROM python:alpine - -RUN echo "*** UID/GID Init. TODO move to a base image ***" -RUN apk add --no-cache shadow -RUN echo "*** create default user ***" && \ - adduser --uid 911 --home /config --shell /bin/false --disabled-password abc && \ - usermod -G users abc -COPY docker/usr/local/sbin/*.sh /usr/local/sbin/ +FROM ajslater/python-alpine:$BASE_VERSION +LABEL version python${BASE_VERSION}_codex-${PKG_VERSION} RUN echo "*** install system runtime packages ***" && \ apk add --no-cache \ @@ -44,5 +39,4 @@ RUN pip3 install --no-index --find-links=/wheels /wheels/codex*.whl VOLUME /comics VOLUME /config EXPOSE 9810 -ENTRYPOINT ["/usr/local/sbin/entrypoint.sh"] CMD ["/usr/local/bin/codex"] diff --git a/Dockerfile.build b/Dockerfile.build index 127e9e0a2..7f9993647 100644 --- a/Dockerfile.build +++ b/Dockerfile.build @@ -2,6 +2,9 @@ FROM ubuntu:focal ARG DEBIAN_FRONTEND=noninteractive +################ +# SYSTEM SETUP # +################ RUN echo "*** install system build dependencies ***" WORKDIR /app RUN apt-get update && apt-get install -y \ @@ -25,6 +28,12 @@ RUN apt-get update && apt-get install -y \ RUN echo "*** install poetry ***" && \ pip3 install -U poetry +# Could make this an image up to here to speed up CI + +############# +# APP SETUP # +############# + RUN echo "**** copying source for dev build ****" COPY . . @@ -38,10 +47,25 @@ RUN echo "*** install frontend build dependencies ***" && \ RUN echo "*** copy choices from frontend ***" && \ cp src/choices/* ../codex/choices/ -# WORKDIR /app -# RUN echo "*** run tests ***" -# RUN ./test.sh -# WORKDIR /app/frontend +######## +# TEST # +######## + +WORKDIR /app +RUN echo "*** run lint ***" +RUN ./lint.sh + +RUN echo "*** run tests ***" +RUN ./test.sh + +VOLUME /test-results +RUN cp test-results/* /test-results + +######### +# BUILD # +######### + +WORKDIR /app/frontend RUN echo "*** build frontend ***" && \ rm -rf ../codex/static_build && \ diff --git a/build-docker-image.sh b/build-docker-image.sh index ea50f8ec3..32f1125d0 100755 --- a/build-docker-image.sh +++ b/build-docker-image.sh @@ -1,3 +1,4 @@ #!/bin/sh # Build a codex docker image suitable for running from Dockerfile -docker-compose build +. ./docker-env +docker build -t "$IMAGE" --build-arg "BASE_VERSION=$BASE_VERSION" --build-arg "PKG_VERSION=$PKG_VERSION" . diff --git a/ci/setup.sh b/ci/setup.sh deleted file mode 100755 index 50bc935c4..000000000 --- a/ci/setup.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# setup script for circleci -set -xeuo pipefail -# TODO Could speed up ci by replacing this with an image - -if [ -n "$CIRCLE_BRANCH" ]; then - sudo apt-get install -y \ - gnupg2 \ - ca-certificates - sudo cp docker/nodesource/nodesource.buster.list /etc/apt/sources.list.d/ - sudo apt-key add ./docker/nodesource/nodesource.gpg.key - sudo apt-get update - sudo apt-get install -y \ - build-essential \ - libffi-dev \ - libjpeg-dev \ - libssl-dev \ - libyaml-dev \ - nodejs \ - python3-pip \ - python3-venv \ - shellcheck \ - zlib1g-dev -fi - -pip3 install -U poetry -poetry update -sudo npm install -g prettier -bash -c "cd frontend && npm install --no-optional" -cp frontend/src/choices/* codex/choices/ diff --git a/docker-compose-build.yaml b/docker-compose-build.yaml index a2080ecfc..00477d12c 100644 --- a/docker-compose-build.yaml +++ b/docker-compose-build.yaml @@ -9,3 +9,4 @@ services: container_name: codex-builder volumes: - ./dist/:/dist/ + - ./test-results/:/app/test-results/ diff --git a/docker-env b/docker-env index 62c8fffa8..4c60e8a65 100644 --- a/docker-env +++ b/docker-env @@ -1,2 +1,5 @@ +IMAGE=ajslater/codex +BASE_VERSION=latest +PKG_VERSION=v0.5.0 PUID=501 PGID=20 diff --git a/docker/usr/local/sbin/entrypoint.sh b/docker/usr/local/sbin/entrypoint.sh deleted file mode 100755 index dbec6ffe7..000000000 --- a/docker/usr/local/sbin/entrypoint.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -set -eu -if [ -n "${PUID:-}${PGID:-}" ]; then - /usr/local/sbin/moduser.sh - su abc --shell /bin/sh --command "$@" -else - exec "$@" -fi diff --git a/docker/usr/local/sbin/moduser.sh b/docker/usr/local/sbin/moduser.sh deleted file mode 100755 index 3d3fff946..000000000 --- a/docker/usr/local/sbin/moduser.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -set -eu -PUID=${PUID:-911} -PGID=${PGID:-911} - -groupmod --non-unique --gid "$PGID" abc -usermod --non-unique --uid "$PUID" abc - -echo " -User uid: $(id -u abc) -User gid: $(id -g abc) -------------------------------------- -" -chown abc:abc /config -chown abc:abc /comics From 8bcbd485d6813aa5e66dd868619bdcb449dc9391 Mon Sep 17 00:00:00 2001 From: AJ Slater Date: Mon, 24 Aug 2020 14:17:29 -0700 Subject: [PATCH 02/11] deploy scripts --- ci/archive-image.sh | 4 ++++ ci/tag-and-push.sh | 9 +++++++++ 2 files changed, 13 insertions(+) create mode 100755 ci/archive-image.sh create mode 100755 ci/tag-and-push.sh diff --git a/ci/archive-image.sh b/ci/archive-image.sh new file mode 100755 index 000000000..a6e78074d --- /dev/null +++ b/ci/archive-image.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -euo pipefail +source ./env +docker save -o "$1" "$IMAGE" diff --git a/ci/tag-and-push.sh b/ci/tag-and-push.sh new file mode 100755 index 000000000..7a2daea89 --- /dev/null +++ b/ci/tag-and-push.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -euo pipefail +source ./env +VERSION=python${BASE_VERSION}_mylar-${PKG_VERSION} +TAG=$VERSION-$(echo "$CIRCLE_SHA1" | head -c 7) +docker tag "$IMAGE" "$IMAGE:$TAG" +docker tag "$IMAGE" "$IMAGE:latest" +docker login -u="$DOCKER_USER" -p="$DOCKER_PASS" +docker push "$REPO" From 1ef4eace049ca9ea6f6874880350512f42bc72f8 Mon Sep 17 00:00:00 2001 From: AJ Slater Date: Mon, 24 Aug 2020 14:21:36 -0700 Subject: [PATCH 03/11] fix requires tag in circleci config yaml --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ec4bee7df..19f3b07d9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -56,7 +56,7 @@ workflows: only: - release - circleci - require: - - build + requires: + - build version: 2.1 From a5e6a691c1027b91f40ac355a084246661b33aa8 Mon Sep 17 00:00:00 2001 From: AJ Slater Date: Mon, 24 Aug 2020 15:05:27 -0700 Subject: [PATCH 04/11] fix lint and docker-env filename --- .circleci/config.yml | 2 +- lint.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 19f3b07d9..4003bd925 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,7 +21,7 @@ jobs: paths: - ./<< pipeline.parameters.image_archive >> - ./dist - - ./env + - ./docker-env - ./ci/tag-and-push.sh root: . deploy: diff --git a/lint.sh b/lint.sh index 4a8741a9a..30ca1c607 100755 --- a/lint.sh +++ b/lint.sh @@ -5,4 +5,4 @@ poetry run isort --check-only --color . poetry run black --check . prettier --check . # hadolint Dockerfile* -shellcheck ./*.sh +shellcheck -x ./*.sh From a319092846bff4f56b3becbd754ef8f98dc0d523 Mon Sep 17 00:00:00 2001 From: AJ Slater Date: Mon, 24 Aug 2020 18:13:09 -0700 Subject: [PATCH 05/11] lint --- README.md | 1 + frontend/src/browser.vue | 12 +++--------- frontend/src/components/auth-dialog.vue | 14 ++++---------- 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 06b29390c..82bf401e6 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,7 @@ Specify a reverse proxy sub path (if you have one) in the config/hypercorn.toml ```toml root_path = "/codex" + ``` #### Nginx Reverse Proxy 502 when container is refreshed. diff --git a/frontend/src/browser.vue b/frontend/src/browser.vue index e2979bc21..9dbf6c373 100644 --- a/frontend/src/browser.vue +++ b/frontend/src/browser.vue @@ -22,9 +22,7 @@ >