From 95657fd61ea7403c6d468a2257e7c56583a51c6b Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Tue, 11 Jan 2022 11:50:01 -0800 Subject: [PATCH 01/13] Don't use git:// protocol for unauthenticated github access. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7cb5c91e38..8676d0ccac 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ repos: -- repo: git://github.com/pre-commit/pre-commit-hooks +- repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.2.1 hooks: - id: trailing-whitespace From 4b7d721e82c3e4722805b0f1de013f8244455d34 Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Tue, 31 Aug 2021 08:41:31 -0700 Subject: [PATCH 02/13] Move minio to dev requirements, update urllib3 --- contentcuration/contentcuration/apps.py | 6 ++++-- .../contentcuration/management/commands/setup.py | 6 ++++-- .../contentcuration/utils/minio_utils.py | 16 ++++++++-------- .../contentcuration/utils/storage_common.py | 10 ++++++++++ requirements-dev.in | 1 + requirements.in | 1 - requirements.txt | 7 +------ 7 files changed, 28 insertions(+), 19 deletions(-) diff --git a/contentcuration/contentcuration/apps.py b/contentcuration/contentcuration/apps.py index e448580629..f42624ce9e 100644 --- a/contentcuration/contentcuration/apps.py +++ b/contentcuration/contentcuration/apps.py @@ -1,7 +1,7 @@ from django.apps import AppConfig from django.conf import settings -from contentcuration.utils.minio_utils import ensure_storage_bucket_public +from contentcuration.utils.storage_common import is_gcs_backend class ContentConfig(AppConfig): @@ -10,5 +10,7 @@ class ContentConfig(AppConfig): def ready(self): # see note in the celery_signals.py file for why we import here. import contentcuration.utils.celery.signals # noqa - if settings.AWS_AUTO_CREATE_BUCKET: + + if settings.AWS_AUTO_CREATE_BUCKET and not is_gcs_backend(): + from contentcuration.utils.minio_utils import ensure_storage_bucket_public ensure_storage_bucket_public() diff --git a/contentcuration/contentcuration/management/commands/setup.py b/contentcuration/contentcuration/management/commands/setup.py index b966f52208..c601870da7 100644 --- a/contentcuration/contentcuration/management/commands/setup.py +++ b/contentcuration/contentcuration/management/commands/setup.py @@ -23,7 +23,7 @@ from contentcuration.utils.db_tools import create_topic from contentcuration.utils.db_tools import create_user from contentcuration.utils.files import duplicate_file -from contentcuration.utils.minio_utils import ensure_storage_bucket_public +from contentcuration.utils.storage_common import is_gcs_backend logmodule.basicConfig() logging = logmodule.getLogger(__name__) @@ -55,7 +55,9 @@ def handle(self, *args, **options): sys.exit() # create the minio bucket - ensure_storage_bucket_public() + if not is_gcs_backend(): + from contentcuration.utils.minio_utils import ensure_storage_bucket_public + ensure_storage_bucket_public() # create the cache table try: diff --git a/contentcuration/contentcuration/utils/minio_utils.py b/contentcuration/contentcuration/utils/minio_utils.py index bb08e06f11..07d1bfe3cb 100644 --- a/contentcuration/contentcuration/utils/minio_utils.py +++ b/contentcuration/contentcuration/utils/minio_utils.py @@ -13,6 +13,8 @@ from minio.error import BucketAlreadyOwnedByYou from minio.error import ResponseError +from contentcuration.utils.storage_common import is_gcs_backend + logger = logging.getLogger(__name__) @@ -42,6 +44,12 @@ def stop_minio(p): def ensure_storage_bucket_public(bucket=None, will_sleep=True): + # GCS' S3 compatibility is broken, especially in bucket operations; + # skip bucket creation there and just bug Aron to create buckets with + # public-read access for you + if is_gcs_backend(): + logging.info("Skipping storage creation on googleapis") + return # If true, sleep for 5 seconds to wait for minio to start if will_sleep: @@ -53,14 +61,6 @@ def ensure_storage_bucket_public(bucket=None, will_sleep=True): bucketname = bucket host = urlparse(settings.AWS_S3_ENDPOINT_URL).netloc - - # GCS' S3 compatibility is broken, especially in bucket operations; - # skip bucket creation there and just bug Aron to create buckets with - # public-read access for you - if "storage.googleapis.com" in host: - logging.info("Skipping storage creation on googleapis") - return - c = minio.Minio( host, access_key=settings.AWS_ACCESS_KEY_ID, diff --git a/contentcuration/contentcuration/utils/storage_common.py b/contentcuration/contentcuration/utils/storage_common.py index b41b018511..9ce747fe1e 100644 --- a/contentcuration/contentcuration/utils/storage_common.py +++ b/contentcuration/contentcuration/utils/storage_common.py @@ -1,6 +1,7 @@ import mimetypes import os from datetime import timedelta +from urllib.parse import urlparse from django.conf import settings from django.core.files.storage import default_storage @@ -19,6 +20,15 @@ class UnknownStorageBackendError(Exception): pass +def is_gcs_backend(): + """ + Determines if storage is GCS backend, which if not we can assume it is minio + :return: A bool + """ + host = urlparse(settings.AWS_S3_ENDPOINT_URL).netloc + return "storage.googleapis.com" in host + + def determine_content_type(filename): """ Guesses the content type of a filename. Returns the mimetype of a file. diff --git a/requirements-dev.in b/requirements-dev.in index bb33682a2b..45eaec50aa 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -40,3 +40,4 @@ git+https://github.com/someshchaturvedi/customizable-django-profiler.git#customi tabulate==0.8.2 fonttools flower==0.9.4 +minio==3.0.3 diff --git a/requirements.in b/requirements.in index abae5cd8a8..3be1692c70 100644 --- a/requirements.in +++ b/requirements.in @@ -14,7 +14,6 @@ newrelic>=2.86.3.70 celery<5 redis pycountry==17.5.14 -minio==3.0.3 pathlib progressbar2==3.38.0 python-postmark==0.5.0 diff --git a/requirements.txt b/requirements.txt index 19715970f9..6d29bcaec8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,6 @@ celery==4.4.7 # via -r requirements.in certifi==2020.12.5 # via - # minio # requests # sentry-sdk cffi==1.14.5 @@ -168,8 +167,6 @@ kombu==4.6.11 # via celery le-utils==0.1.31 # via -r requirements.in -minio==3.0.3 - # via -r requirements.in newrelic==6.2.0.156 # via -r requirements.in oauth2client==4.1.3 @@ -232,7 +229,6 @@ pytz==2021.1 # django # django-postmark # google-api-core - # minio raven==6.10.0 # via -r requirements.in redis==3.5.3 @@ -278,10 +274,9 @@ typing-extensions==3.10.0.0 # via asgiref uritemplate==3.0.1 # via google-api-python-client -urllib3==1.26.4 +urllib3==1.26.5 # via # botocore - # minio # requests # sentry-sdk vine==1.3.0 From 99c04d59dffc9b0405ed42bedb87b23e4a14c9d5 Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Tue, 31 Aug 2021 08:46:46 -0700 Subject: [PATCH 03/13] Update dev requirements --- requirements-dev.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0d03c993d8..ef41dbe4db 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -40,6 +40,7 @@ certifi==2020.12.5 # via # -c requirements.txt # geventhttpclient + # minio # requests cfgv==3.3.0 # via pre-commit @@ -194,6 +195,8 @@ mccabe==0.6.1 # via # flake8 # pylint +minio==3.0.3 + # via -r requirements-dev.in mixer==6.1.3 # via -r requirements-dev.in mock==4.0.3 @@ -301,6 +304,7 @@ pytz==2021.1 # celery # django # flower + # minio pyyaml==5.4.1 # via # aspy.yaml @@ -369,9 +373,10 @@ uritemplate==3.0.1 # -c requirements.txt # coreapi # drf-yasg -urllib3==1.26.4 +urllib3==1.26.5 # via # -c requirements.txt + # minio # requests vine==1.3.0 # via From 45632d1c5e4516a34681caf43a3048bd13216e4f Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Thu, 18 Nov 2021 05:35:25 +0530 Subject: [PATCH 04/13] chore: minio 3.0.3 -> 7.1.1 --- .../contentcuration/utils/minio_utils.py | 61 ++++++++++++------- requirements-dev.in | 2 +- requirements-dev.txt | 7 ++- 3 files changed, 44 insertions(+), 26 deletions(-) diff --git a/contentcuration/contentcuration/utils/minio_utils.py b/contentcuration/contentcuration/utils/minio_utils.py index 07d1bfe3cb..559d09b417 100644 --- a/contentcuration/contentcuration/utils/minio_utils.py +++ b/contentcuration/contentcuration/utils/minio_utils.py @@ -1,20 +1,20 @@ from future import standard_library + standard_library.install_aliases() import atexit import logging import multiprocessing import subprocess import time +import json from urllib.parse import urlparse import minio from django.conf import settings -from minio import policy -from minio.error import BucketAlreadyOwnedByYou -from minio.error import ResponseError from contentcuration.utils.storage_common import is_gcs_backend + logger = logging.getLogger(__name__) @@ -65,20 +65,34 @@ def ensure_storage_bucket_public(bucket=None, will_sleep=True): host, access_key=settings.AWS_ACCESS_KEY_ID, secret_key=settings.AWS_SECRET_ACCESS_KEY, - secure=False + secure=False, ) + READ_ONLY_POLICY = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": {"AWS": "*"}, + "Action": ["s3:GetBucketLocation", "s3:ListBucket"], + "Resource": "arn:aws:s3:::{bucketname}".format(bucketname=bucketname), + }, + { + "Effect": "Allow", + "Principal": {"AWS": "*"}, + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::{bucketname}/*".format(bucketname=bucketname), + }, + ], + } + if not c.bucket_exists(bucketname): - try: - c.make_bucket(bucketname) - except BucketAlreadyOwnedByYou: - pass + c.make_bucket(bucketname) - try: - c.set_bucket_policy(bucketname, "", policy.Policy.READ_ONLY) - logger.debug("Successfully set the bucket policy to read only!") - except ResponseError as e: - logger.warning("Error setting bucket {} to readonly: {}".format(bucket, e)) + c.set_bucket_policy( + bucketname, + json.dumps(READ_ONLY_POLICY), + ) def ensure_bucket_deleted(bucket=None): @@ -94,21 +108,24 @@ def ensure_bucket_deleted(bucket=None): # skip bucket creation there and just bug Aron to create buckets with # public-read access for you if "storage.googleapis.com" in host: - logging.info("Skipping storage deletion on googleapis; that sounds like a production bucket!") + logging.info( + "Skipping storage deletion on googleapis; that sounds like a production bucket!" + ) return minio_client = minio.Minio( host, access_key=settings.AWS_ACCESS_KEY_ID, secret_key=settings.AWS_SECRET_ACCESS_KEY, - secure=False + secure=False, ) if minio_client.bucket_exists(bucketname): - try: - # We need to delete all objects first, before we can actually delete the bucket. - objs = (o.object_name for o in minio_client.list_objects(bucketname, recursive=True)) - list(minio_client.remove_objects(bucketname, objs)) # evaluate the generator, or else remove_objects won't actually execute - minio_client.remove_bucket(bucketname) - except BucketAlreadyOwnedByYou: - pass + # We need to delete all objects first, before we can actually delete the bucket. + objs = ( + o.object_name for o in minio_client.list_objects(bucketname, recursive=True) + ) + list( + minio_client.remove_objects(bucketname, objs) + ) # evaluate the generator, or else remove_objects won't actually execute + minio_client.remove_bucket(bucketname) diff --git a/requirements-dev.in b/requirements-dev.in index 45eaec50aa..ddb1356b3a 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -40,4 +40,4 @@ git+https://github.com/someshchaturvedi/customizable-django-profiler.git#customi tabulate==0.8.2 fonttools flower==0.9.4 -minio==3.0.3 +minio==7.1.1 diff --git a/requirements-dev.txt b/requirements-dev.txt index ef41dbe4db..b9cf415e9d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -195,7 +195,7 @@ mccabe==0.6.1 # via # flake8 # pylint -minio==3.0.3 +minio==7.1.1 # via -r requirements-dev.in mixer==6.1.3 # via -r requirements-dev.in @@ -304,7 +304,6 @@ pytz==2021.1 # celery # django # flower - # minio pyyaml==5.4.1 # via # aspy.yaml @@ -361,7 +360,9 @@ traitlets==4.3.3 typed-ast==1.4.3 # via astroid typing-extensions==3.10.0.0 - # via asgiref + # via + # -c requirements.txt + # asgiref ujson==4.0.2 # via # python-jsonrpc-server From 9b341111dac2d2d4c3b13273aa6759b10f9773ee Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Thu, 18 Nov 2021 06:10:16 +0530 Subject: [PATCH 05/13] chore: remove ipdb --- requirements-dev.in | 1 - requirements-dev.txt | 29 ----------------------------- 2 files changed, 30 deletions(-) diff --git a/requirements-dev.in b/requirements-dev.in index ddb1356b3a..c74804745c 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,5 +1,4 @@ -c requirements.txt -ipdb python-language-server django-concurrent-test-helper==0.7.0 django-debug-panel==0.8.3 diff --git a/requirements-dev.txt b/requirements-dev.txt index b9cf415e9d..6ff55ed095 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -26,8 +26,6 @@ autoflake==1.4 # via -r requirements-dev.in autopep8==1.4 # via -r requirements-dev.in -backcall==0.2.0 - # via ipython billiard==3.6.4.0 # via # -c requirements.txt @@ -73,10 +71,6 @@ git+https://github.com/someshchaturvedi/customizable-django-profiler.git#customi # via -r requirements-dev.in dataclasses==0.8 # via werkzeug -decorator==5.0.9 - # via - # ipython - # traitlets distlib==0.3.1 # via virtualenv django-concurrent-test-helper==0.7.0 @@ -154,12 +148,6 @@ inflection==0.5.1 # via drf-yasg iniconfig==1.1.1 # via pytest -ipdb==0.13.7 - # via -r requirements-dev.in -ipython-genutils==0.2.0 - # via traitlets -ipython==7.16.1 - # via ipdb isort==5.8.0 # via # -r requirements-dev.in @@ -172,7 +160,6 @@ itypes==1.2.0 jedi==0.17.2 # via # -r requirements-dev.in - # ipython # python-language-server # python-lsp-server jinja2==3.0.1 @@ -218,10 +205,6 @@ parso==0.7.1 # via jedi pep517==0.10.0 # via pip-tools -pexpect==4.8.0 - # via ipython -pickleshare==0.7.5 - # via ipython pip-tools==6.1.0 # via -r requirements-dev.in pluggy==0.13.1 @@ -231,12 +214,8 @@ pluggy==0.13.1 # python-lsp-server pre-commit==1.15.1 # via -r requirements-dev.in -prompt-toolkit==3.0.18 - # via ipython psutil==5.8.0 # via locust -ptyprocess==0.7.0 - # via pexpect py==1.10.0 # via pytest pycodestyle==2.3.1 @@ -247,8 +226,6 @@ pyflakes==1.5.0 # via # autoflake # flake8 -pygments==2.9.0 - # via ipython pyinstrument-cext==0.2.4 # via pyinstrument pyinstrument==3.4.2 @@ -332,7 +309,6 @@ six==1.16.0 # geventhttpclient # pre-commit # python-dateutil - # traitlets # virtualenv sqlparse==0.4.1 # via @@ -348,15 +324,12 @@ text-unidecode==1.2 toml==0.10.2 # via # coverage - # ipdb # pep517 # pre-commit # pylint # pytest tornado==6.1 # via flower -traitlets==4.3.3 - # via ipython typed-ast==1.4.3 # via astroid typing-extensions==3.10.0.0 @@ -388,8 +361,6 @@ virtualenv==20.4.6 # via pre-commit watchdog==2.1.2 # via pytest-watch -wcwidth==0.2.5 - # via prompt-toolkit werkzeug==2.0.1 # via # flask From bb6e2d7955ab0a5afb31d4fdfceb0266dcecb1e2 Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Sat, 20 Nov 2021 22:29:17 +0530 Subject: [PATCH 06/13] tests: use the new remove object api & cleanup unused functions --- .../contentcuration/utils/minio_utils.py | 40 ++++--------------- 1 file changed, 7 insertions(+), 33 deletions(-) diff --git a/contentcuration/contentcuration/utils/minio_utils.py b/contentcuration/contentcuration/utils/minio_utils.py index 559d09b417..b7d0a7d62b 100644 --- a/contentcuration/contentcuration/utils/minio_utils.py +++ b/contentcuration/contentcuration/utils/minio_utils.py @@ -1,10 +1,8 @@ from future import standard_library standard_library.install_aliases() -import atexit + import logging -import multiprocessing -import subprocess import time import json from urllib.parse import urlparse @@ -18,31 +16,6 @@ logger = logging.getLogger(__name__) -def start_minio(): - """ - Start a minio subprocess, controlled by another thread. - - Returns the daemonized thread controlling the minio subprocess. - """ - minio_process = multiprocessing.Process(target=_start_minio) - minio_process.start() - atexit.register(lambda: stop_minio(minio_process)) - return minio_process - - -def _start_minio(): - logger.info("Starting minio") - - subprocess.Popen( - ["run_minio.py"], - stdin=subprocess.PIPE, - ) - - -def stop_minio(p): - p.terminate() - - def ensure_storage_bucket_public(bucket=None, will_sleep=True): # GCS' S3 compatibility is broken, especially in bucket operations; # skip bucket creation there and just bug Aron to create buckets with @@ -107,7 +80,7 @@ def ensure_bucket_deleted(bucket=None): # GCS' S3 compatibility is broken, especially in bucket operations; # skip bucket creation there and just bug Aron to create buckets with # public-read access for you - if "storage.googleapis.com" in host: + if is_gcs_backend(): logging.info( "Skipping storage deletion on googleapis; that sounds like a production bucket!" ) @@ -122,10 +95,11 @@ def ensure_bucket_deleted(bucket=None): if minio_client.bucket_exists(bucketname): # We need to delete all objects first, before we can actually delete the bucket. - objs = ( + objs_name = ( o.object_name for o in minio_client.list_objects(bucketname, recursive=True) ) - list( - minio_client.remove_objects(bucketname, objs) - ) # evaluate the generator, or else remove_objects won't actually execute + + for o in objs_name: + minio_client.remove_object(bucketname, o) + minio_client.remove_bucket(bucketname) From a47396f2133ff604a8081f5a3faab18be07ed100 Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Wed, 9 Mar 2022 01:19:00 +0530 Subject: [PATCH 07/13] feat: welcome postgres 12! --- .github/workflows/pythontest.yml | 2 +- README.md | 2 +- docker-compose.yml | 2 +- docs/manual_setup.md | 14 ++++++-------- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 8544989bf6..096aae7269 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -24,7 +24,7 @@ jobs: # Label used to access the service container postgres: # Docker Hub image - image: postgres + image: postgres:12.10 # Provide the password for postgres env: POSTGRES_USER: learningequality diff --git a/README.md b/README.md index 4bbb1f9a81..440b29d1bd 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ To confirm that the services are running, run `docker ps`, and you should see th CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES e09c5c203b93 redis:4.0.9 "docker-entrypoint.s…" 51 seconds ago Up 49 seconds 0.0.0.0:6379->6379/tcp studio_vue-refactor_redis_1 6164371efb6b minio/minio "minio server /data" 51 seconds ago Up 49 seconds 0.0.0.0:9000->9000/tcp studio_vue-refactor_minio_1 -c86bbfa3a59e postgres:9.6 "docker-entrypoint.s…" 51 seconds ago Up 49 seconds 0.0.0.0:5432->5432/tcp studio_vue-refactor_postgres_1 +c86bbfa3a59e postgres:12.10 "docker-entrypoint.s…" 51 seconds ago Up 49 seconds 0.0.0.0:5432->5432/tcp studio_vue-refactor_postgres_1 ``` diff --git a/docker-compose.yml b/docker-compose.yml index cb1cc9decb..8b2bb69209 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,7 +53,7 @@ services: - minio_data:/data postgres: - image: postgres:9.6 + image: postgres:12.10 environment: PGDATA: /var/lib/postgresql/data/pgdata POSTGRES_USER: learningequality diff --git a/docs/manual_setup.md b/docs/manual_setup.md index c61e0cc082..0db63b8a65 100644 --- a/docs/manual_setup.md +++ b/docs/manual_setup.md @@ -1,5 +1,3 @@ - - # Manually installing dependencies ## Install @@ -9,7 +7,7 @@ Rather than using Docker, it is also possible to manually install the dependenci - [python (2.7)](https://www.python.org/downloads/release/python-2713/) - [python-pip](https://pip.pypa.io/en/stable/installing/) - [nodejs (10.x)](https://nodejs.org/en/download/) - - [Postgres DB](https://www.postgresql.org/download/) + - [Postgres DB (12.10)](https://www.postgresql.org/download/) - [redis](https://redis.io/topics/quickstart) - [minio server](https://www.minio.io/downloads.html) - [nginx](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/) @@ -37,7 +35,7 @@ curl -sL https://deb.nodesource.com/setup_10.x | bash - # Install packages apt-get install -y python python-pip python-dev python-tk \ - postgresql-server-dev-all postgresql-contrib postgresql-client postgresql \ + postgresql-server-dev-all postgresql-contrib postgresql-client postgresql-12.10 \ ffmpeg nodejs libmagickwand-dev nginx redis-server wkhtmltopdf ``` @@ -46,9 +44,9 @@ apt-get install -y python python-pip python-dev python-tk \ You can install the corresponding packages using Homebrew: ```bash -brew install postgresql@9.6 redis node ffmpeg imagemagick@6 gs +brew install postgresql@12.10 redis node ffmpeg imagemagick@6 gs brew install minio/stable/minio -brew link --force postgresql@9.6 +brew link --force postgresql@12.10 brew link --force imagemagick@6 ``` @@ -62,13 +60,13 @@ Windows is no longer supported due to incompatibilities with some of the require ## Set up the database -Install [postgres](https://www.postgresql.org/download/) if you don't have it already. If you're using a package manager, you need to make sure you install the following packages: `postgresql`, `postgresql-contrib`, and `postgresql-server-dev-all` which will be required to build `psycopg2` python driver. +Install [postgres](https://www.postgresql.org/download/) if you don't have it already. If you're using a package manager, you need to make sure you install the following packages: `postgresql-12.10`, `postgresql-contrib`, and `postgresql-server-dev-all` which will be required to build `psycopg2` python driver. Make sure postgres is running: ```bash service postgresql start -# alternatively: pg_ctl -D /usr/local/var/postgresql@9.6 start +# alternatively: pg_ctl -D /usr/local/var/postgresql@12.10 start ``` Start the client with: From 3ce3548ec159bb40dfd06f779e024eb47764662c Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Wed, 9 Mar 2022 02:12:16 +0530 Subject: [PATCH 08/13] Pin CI services version to same as docker-compose services --- .github/workflows/pythontest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 096aae7269..71f1663235 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -42,7 +42,7 @@ jobs: # Label used to access the service container redis: # Docker Hub image - image: redis + image: redis:4.0.9 # Set health checks to wait until redis has started options: >- --health-cmd "redis-cli ping" @@ -53,7 +53,7 @@ jobs: # Maps port 6379 on service container to the host - 6379:6379 minio: - image: bitnami/minio + image: minio/minio:RELEASE.2020-06-22T03-12-50Z env: MINIO_API_CORS_ALLOW_ORIGIN: http://localhost:8080 MINIO_ACCESS_KEY: development From 35e2f95802ba7a8c86e06e8b89944518ec0e70ec Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Thu, 10 Mar 2022 23:22:57 +0530 Subject: [PATCH 09/13] fix: pin postgres major & use minio/minio on CI --- .github/workflows/pythontest.yml | 31 ++++++++++++++++++++++--------- docker-compose.yml | 2 +- docs/manual_setup.md | 12 ++++++------ 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 71f1663235..6c4f1f3aa7 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -15,16 +15,18 @@ jobs: with: github_token: ${{ github.token }} paths: '["**.py", "requirements.txt", "requirements-dev.txt", ".github/workflows/pythontest.yml"]' + unit_test: name: Python unit tests needs: pre_job if: ${{ needs.pre_job.outputs.should_skip != 'true' }} runs-on: ubuntu-latest + services: # Label used to access the service container postgres: # Docker Hub image - image: postgres:12.10 + image: postgres:12 # Provide the password for postgres env: POSTGRES_USER: learningequality @@ -52,16 +54,22 @@ jobs: ports: # Maps port 6379 on service container to the host - 6379:6379 - minio: - image: minio/minio:RELEASE.2020-06-22T03-12-50Z - env: - MINIO_API_CORS_ALLOW_ORIGIN: http://localhost:8080 - MINIO_ACCESS_KEY: development - MINIO_SECRET_KEY: development - ports: - - 9000:9000 steps: - uses: actions/checkout@v2 + - name: Set up minio + run: | + docker run -d -p 9000:9000 --name minio \ + -e "MINIO_ACCESS_KEY=minioadmin" \ + -e "MINIO_SECRET_KEY=minioadmin" \ + -v /tmp/minio_data:/data \ + -v /tmp/minio_config:/root/.minio \ + minio/minio server /data + + export AWS_ACCESS_KEY_ID=minioadmin + export AWS_SECRET_ACCESS_KEY=minioadmin + export AWS_EC2_METADATA_DISABLED=true + + aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://testbucket - name: Set up Python 3.6 uses: actions/setup-python@v2 with: @@ -79,6 +87,11 @@ jobs: pip install pip-tools pip-sync requirements.txt requirements-dev.txt - name: Test pytest + env: + AWS_BUCKET_NAME: testbucket + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + AWS_S3_ENDPOINT_URL: http://127.0.0.1:9000 run: | sh -c './contentcuration/manage.py makemigrations --check' pytest diff --git a/docker-compose.yml b/docker-compose.yml index 8b2bb69209..02732ba585 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,7 +53,7 @@ services: - minio_data:/data postgres: - image: postgres:12.10 + image: postgres:12 environment: PGDATA: /var/lib/postgresql/data/pgdata POSTGRES_USER: learningequality diff --git a/docs/manual_setup.md b/docs/manual_setup.md index 0db63b8a65..f0e3c77cf7 100644 --- a/docs/manual_setup.md +++ b/docs/manual_setup.md @@ -7,7 +7,7 @@ Rather than using Docker, it is also possible to manually install the dependenci - [python (2.7)](https://www.python.org/downloads/release/python-2713/) - [python-pip](https://pip.pypa.io/en/stable/installing/) - [nodejs (10.x)](https://nodejs.org/en/download/) - - [Postgres DB (12.10)](https://www.postgresql.org/download/) + - [Postgres DB (12.x)](https://www.postgresql.org/download/) - [redis](https://redis.io/topics/quickstart) - [minio server](https://www.minio.io/downloads.html) - [nginx](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/) @@ -35,7 +35,7 @@ curl -sL https://deb.nodesource.com/setup_10.x | bash - # Install packages apt-get install -y python python-pip python-dev python-tk \ - postgresql-server-dev-all postgresql-contrib postgresql-client postgresql-12.10 \ + postgresql-server-dev-all postgresql-contrib postgresql-client postgresql-12 \ ffmpeg nodejs libmagickwand-dev nginx redis-server wkhtmltopdf ``` @@ -44,9 +44,9 @@ apt-get install -y python python-pip python-dev python-tk \ You can install the corresponding packages using Homebrew: ```bash -brew install postgresql@12.10 redis node ffmpeg imagemagick@6 gs +brew install postgresql@12 redis node ffmpeg imagemagick@6 gs brew install minio/stable/minio -brew link --force postgresql@12.10 +brew link --force postgresql@12 brew link --force imagemagick@6 ``` @@ -60,13 +60,13 @@ Windows is no longer supported due to incompatibilities with some of the require ## Set up the database -Install [postgres](https://www.postgresql.org/download/) if you don't have it already. If you're using a package manager, you need to make sure you install the following packages: `postgresql-12.10`, `postgresql-contrib`, and `postgresql-server-dev-all` which will be required to build `psycopg2` python driver. +Install [postgres](https://www.postgresql.org/download/) if you don't have it already. If you're using a package manager, you need to make sure you install the following packages: `postgresql-12`, `postgresql-contrib`, and `postgresql-server-dev-all` which will be required to build `psycopg2` python driver. Make sure postgres is running: ```bash service postgresql start -# alternatively: pg_ctl -D /usr/local/var/postgresql@12.10 start +# alternatively: pg_ctl -D /usr/local/var/postgresql@12 start ``` Start the client with: From 94478d924aee91b31b5c565ed138df9281eeefca Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Thu, 10 Mar 2022 23:28:19 +0530 Subject: [PATCH 10/13] fix: unnecessary exports removed from minio CI --- .github/workflows/pythontest.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 6c4f1f3aa7..9b26f09ba8 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -64,11 +64,6 @@ jobs: -v /tmp/minio_data:/data \ -v /tmp/minio_config:/root/.minio \ minio/minio server /data - - export AWS_ACCESS_KEY_ID=minioadmin - export AWS_SECRET_ACCESS_KEY=minioadmin - export AWS_EC2_METADATA_DISABLED=true - aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://testbucket - name: Set up Python 3.6 uses: actions/setup-python@v2 From a963f973a27fab4b6e941b2d51662b5cfc1837a1 Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Thu, 10 Mar 2022 23:31:35 +0530 Subject: [PATCH 11/13] fix: CI --- .github/workflows/pythontest.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 9b26f09ba8..966dd9613f 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -58,13 +58,16 @@ jobs: - uses: actions/checkout@v2 - name: Set up minio run: | - docker run -d -p 9000:9000 --name minio \ - -e "MINIO_ACCESS_KEY=minioadmin" \ - -e "MINIO_SECRET_KEY=minioadmin" \ - -v /tmp/minio_data:/data \ - -v /tmp/minio_config:/root/.minio \ - minio/minio server /data - aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://testbucket + docker run -d -p 9000:9000 --name minio \ + -e "MINIO_ACCESS_KEY=minioadmin" \ + -e "MINIO_SECRET_KEY=minioadmin" \ + -v /tmp/minio_data:/data \ + -v /tmp/minio_config:/root/.minio \ + minio/minio server /data + export AWS_ACCESS_KEY_ID=minioadmin + export AWS_SECRET_ACCESS_KEY=minioadmin + export AWS_EC2_METADATA_DISABLED=true + aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://testbucket - name: Set up Python 3.6 uses: actions/setup-python@v2 with: From 5b32ef9715c68a41e370cee15d4e8fa2798931c9 Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Thu, 10 Mar 2022 23:35:55 +0530 Subject: [PATCH 12/13] fix: Blaine's elegant solution! --- .github/workflows/pythontest.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 966dd9613f..7fdf6e178c 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -59,15 +59,11 @@ jobs: - name: Set up minio run: | docker run -d -p 9000:9000 --name minio \ - -e "MINIO_ACCESS_KEY=minioadmin" \ - -e "MINIO_SECRET_KEY=minioadmin" \ + -e "MINIO_ACCESS_KEY=development" \ + -e "MINIO_SECRET_KEY=development" \ -v /tmp/minio_data:/data \ -v /tmp/minio_config:/root/.minio \ minio/minio server /data - export AWS_ACCESS_KEY_ID=minioadmin - export AWS_SECRET_ACCESS_KEY=minioadmin - export AWS_EC2_METADATA_DISABLED=true - aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://testbucket - name: Set up Python 3.6 uses: actions/setup-python@v2 with: @@ -85,11 +81,6 @@ jobs: pip install pip-tools pip-sync requirements.txt requirements-dev.txt - name: Test pytest - env: - AWS_BUCKET_NAME: testbucket - AWS_ACCESS_KEY_ID: minioadmin - AWS_SECRET_ACCESS_KEY: minioadmin - AWS_S3_ENDPOINT_URL: http://127.0.0.1:9000 run: | sh -c './contentcuration/manage.py makemigrations --check' pytest From 66dca07d6b08b14af1186d93b205fcb026b5147d Mon Sep 17 00:00:00 2001 From: Aron Fyodor Asor <191955+aronasorman@users.noreply.github.com> Date: Thu, 24 Mar 2022 14:50:16 -0500 Subject: [PATCH 13/13] fix: use the jspdf https url instead of git:// (#3349) * fix: use the jspdf https url instead of git:// git:// started failing recently, and no docker build can happen * fixup! fix: use the jspdf https url instead of git:// Co-authored-by: Aron Asor --- package.json | 2 +- yarn.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index a7d64edda6..89431656f0 100644 --- a/package.json +++ b/package.json @@ -79,7 +79,7 @@ "html2canvas": "^1.0.0-rc.5", "i18n-iso-countries": "^5.1.0", "jquery": "^2.2.4", - "jspdf": "git://github.com/MrRio/jsPDF.git#b7a1d8239c596292ce86dafa77f05987bcfa2e6e", + "jspdf": "https://github.com/MrRio/jsPDF.git#b7a1d8239c596292ce86dafa77f05987bcfa2e6e", "jszip": "^2.5.0", "jszip-utils": "0.0.2", "kolibri-tools": "^0.14.5-dev.4", diff --git a/yarn.lock b/yarn.lock index 6c3e36512c..8da1fa3deb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12320,9 +12320,9 @@ jsonparse@^1.2.0: resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" integrity sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA= -"jspdf@git://github.com/MrRio/jsPDF.git#b7a1d8239c596292ce86dafa77f05987bcfa2e6e": +"jspdf@https://github.com/MrRio/jsPDF.git#b7a1d8239c596292ce86dafa77f05987bcfa2e6e": version "2.1.1" - resolved "git://github.com/MrRio/jsPDF.git#b7a1d8239c596292ce86dafa77f05987bcfa2e6e" + resolved "https://github.com/MrRio/jsPDF.git#b7a1d8239c596292ce86dafa77f05987bcfa2e6e" dependencies: atob "^2.1.2" btoa "^1.2.1"