From 887dd7c2a8370ab1a7c4366b9fe3f5680dbbd9b3 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 6 Sep 2022 18:02:48 +0200 Subject: [PATCH 1/2] Support installing charms with built wheels The file naming convention for Python source wheels and built binary wheels are quite different. Update the code so that it can extract package name and versions from both wheel types. Signed-off-by: Frode Nordahl --- lib/charms/layer/basic.py | 15 +++++++- unit_tests/test_lib_charms_layer_basic.py | 43 ++++++++++++++++++++--- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/lib/charms/layer/basic.py b/lib/charms/layer/basic.py index 03eead4..3e741fa 100644 --- a/lib/charms/layer/basic.py +++ b/lib/charms/layer/basic.py @@ -221,6 +221,8 @@ def bootstrap_charm_deps(): # if not cfg.get('use_venv', True) and pre_eoan: if not cfg.get('use_venv', True): reinstall_flag = '--ignore-installed' + if not pkgs: + continue check_call([pip, 'install', '-U', reinstall_flag, '--no-index', '--no-cache-dir', '-f', 'wheelhouse'] + list(pkgs), env=_get_subprocess_env()) @@ -290,7 +292,18 @@ def _load_installed_versions(pip): def _load_wheelhouse_versions(): versions = {} for wheel in glob('wheelhouse/*'): - pkg, ver = os.path.basename(wheel).rsplit('-', 1) + if wheel.endswith('.whl'): + # The binary wheel package format has a more stringent definition + # of how the filenames are formulated. As such we can safely + # extract the exact version string and store that. + # + # Reference: + # PEP 427 https://peps.python.org/pep-0427/#file-name-convention + # 'setuptools_scm-6.4.2-py3-none-any.whl'.split('-', 2) == + # ['setuptools_scm', '6.4.2', 'py3-none-any.whl'] + pkg, ver, _ = os.path.basename(wheel).split('-', 2) + else: + pkg, ver = os.path.basename(wheel).rsplit('-', 1) # nb: LooseVersion ignores the file extension versions[pkg.replace('_', '-')] = LooseVersion(ver) return versions diff --git a/unit_tests/test_lib_charms_layer_basic.py b/unit_tests/test_lib_charms_layer_basic.py index b1e6cfe..98db417 100644 --- a/unit_tests/test_lib_charms_layer_basic.py +++ b/unit_tests/test_lib_charms_layer_basic.py @@ -11,25 +11,58 @@ class TestLayerBasic(test_utils.BaseTestCase): + wheelhouse_glob = [ + 'python-dateutil-2.8.1.tar.gz', + 'setuptools_scm-1.17.0.tar.gz', + 'wheel-0.33.6.tar.gz', + 'cffi-1.15.1' + '-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl', + 'flit_core-3.7.1-py3-none-any.whl', + ] + def test__load_wheelhouse_versions(self): self.patch_object(basic, 'glob') self.patch_object(basic, 'LooseVersion') - self.glob.return_value = [ - 'python-dateutil-2.8.1.tar.gz', - 'setuptools_scm-1.17.0.tar.gz', - 'wheel-0.33.6.tar.gz', - ] + self.glob.return_value = self.wheelhouse_glob self.assertDictEqual( basic._load_wheelhouse_versions(), { 'setuptools-scm': mock.ANY, 'python-dateutil': mock.ANY, 'wheel': mock.ANY, + 'cffi': mock.ANY, + 'flit-core': mock.ANY, }) self.LooseVersion.assert_has_calls([ mock.call('0.33.6.tar.gz'), mock.call('2.8.1.tar.gz'), mock.call('1.17.0.tar.gz'), + mock.call('1.15.1'), + mock.call('3.7.1'), ], any_order=True) + self.assertEqual( + self.LooseVersion.call_count, + 5) + + def test__add_back_versions(self): + self.patch_object(basic, 'glob') + self.glob.return_value = self.wheelhouse_glob + self.assertEqual( + basic._add_back_versions( + [ + 'python-dateutil', + 'setuptools-scm', + 'wheel', + 'cffi', + 'flit-core', + ], + basic._load_wheelhouse_versions()), + [ + 'python-dateutil==2.8.1', + 'setuptools-scm==1.17.0', + 'wheel==0.33.6', + 'cffi==1.15.1', + 'flit-core==3.7.1' + ]) @patch.dict('os.environ', {'LANG': 'su_SU.UTF-8'}) def test__get_subprocess_env_lang_set(self): From 845e813cf06e92b02135dc2d2fb9d26b63901381 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 7 Sep 2022 10:31:44 +0200 Subject: [PATCH 2/2] binary-wheels: Add functional test Signed-off-by: Frode Nordahl --- tests/bundles/minimal.yaml | 4 ++++ tox.ini | 2 ++ 2 files changed, 6 insertions(+) diff --git a/tests/bundles/minimal.yaml b/tests/bundles/minimal.yaml index a9ce013..4d6dd2b 100644 --- a/tests/bundles/minimal.yaml +++ b/tests/bundles/minimal.yaml @@ -17,6 +17,10 @@ applications: series: focal charm: /tmp/charm-builds/minimal num_units: 1 + minimal-binary-wheels-focal: + series: focal + charm: /tmp/charm-builds/minimal-binary-wheels + num_units: 1 #minimal-no-venv-trusty: #series: trusty #charm: /tmp/charm-builds/minimal-no-venv diff --git a/tox.ini b/tox.ini index d8e4365..b6c5b72 100644 --- a/tox.ini +++ b/tox.ini @@ -20,11 +20,13 @@ setenv = CHARM_LAYERS_DIR=/tmp/charm-builds/_tmp/layers passenv = HOME commands = /bin/rm -rf /tmp/charm-builds/_tmp /tmp/charm-builds/minimal + /bin/rm -rf /tmp/charm-builds/_tmp /tmp/charm-builds/minimal-binary-wheels /bin/rm -rf /tmp/charm-builds/_tmp /tmp/charm-builds/minimal-no-venv /bin/mkdir -p /tmp/charm-builds/_tmp/layers /bin/bash -c '/bin/ln -sf $(readlink --canonicalize {toxinidir}) /tmp/charm-builds/_tmp/layers/layer-basic' /bin/bash -c '/bin/ln -sf $(readlink --canonicalize {toxinidir}/tests/charm-minimal) /tmp/charm-builds/_tmp/layers/charm-minimal' charm-build --log-level DEBUG tests/charm-minimal + charm-build --log-level DEBUG --binary-wheels -n minimal-binary-wheels tests/charm-minimal charm-build --log-level DEBUG tests/charm-minimal-no-venv functest-run-suite --keep-model