From e3b9c4165a1db60e12bc7fb8f6cc004426bdabd2 Mon Sep 17 00:00:00 2001 From: Alain Lichnewsky Date: Wed, 21 Apr 2021 23:59:26 +0200 Subject: [PATCH 1/3] Corrects 2 issues concerning Python2.7 : - dpath.segment.leaf failing to recognize unicode - collections.abc included in collections in Python2 Tests added in tests/test_unicode.py, restructured to also operate autonomously like unittest --- .github/workflows/linterTest.yml | 56 ++++++++++++++ .github/workflows/python3Test.yml | 71 ++++++++++++++++++ dpath/segments.py | 9 ++- dpath/util.py | 10 ++- tests/test_unicode.py | 121 ++++++++++++++++++++++++------ 5 files changed, 241 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/linterTest.yml create mode 100644 .github/workflows/python3Test.yml diff --git a/.github/workflows/linterTest.yml b/.github/workflows/linterTest.yml new file mode 100644 index 0000000..1f284eb --- /dev/null +++ b/.github/workflows/linterTest.yml @@ -0,0 +1,56 @@ +name: LinterTest + # ------------------------------------------------------------ + # (C) Alain Lichnewsky, 2021 + # + # For running under Github's Actions + # + # Script performs basic static test of the package under Python-3, + # including added functionality. + # ------------------------------------------------------------ + +# Controls when the action will run. +on: + # + ## Not enabled, would triggers the workflow on push or pull request events but only + ## for the AL-addRegexp branch. + #push: + # branches: [ AL-addRegexp ] + + # Allows to run this workflow manually from the Github Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "super-lint" + super-lint: + # Steps represent a sequence of tasks that will be executed as part of the job + # Name the Job + name: Lint code base + # Set the type of machine to run on + runs-on: ubuntu-latest + + steps: + # Checks out a copy of your repository on the ubuntu-latest machine + - name: Checkout code + uses: actions/checkout@v2 + + # Runs a single command using the runners shell, in practice it is useful + # to figure out some of the environment setup + - name: Use shell to figure out environment + run: echo Hello from Github Actions !!; + bash --version | head -1 ; + echo LANG=${LANG} SHELL=${SHELL} ; + echo PATH=${PATH} ; + pwd; + ls -ltha; + + # Runs the Super-Linter action + - name: Run Super-Linter + uses: github/super-linter@v3 + # + # this script requires some environment variables + # + env: + DEFAULT_BRANCH: AL-addRegexp + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + diff --git a/.github/workflows/python3Test.yml b/.github/workflows/python3Test.yml new file mode 100644 index 0000000..929f4db --- /dev/null +++ b/.github/workflows/python3Test.yml @@ -0,0 +1,71 @@ +name: Test python package dpath-python + # ------------------------------------------------------------ + # (C) Alain Lichnewsky, 2021 + # + # For running under Github's Actions + # + # Script performs basic test of the Python-3 version of the package + # including added functionality. + # ------------------------------------------------------------ + +on: #[push] + workflow_dispatch: + +jobs: + test-python3: + + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: '3.8' + architecture: 'x64' + + - name: Ascertain configuration + # + # Collect information concerning $HOME + # location of file loaded from Github/ + run: | + echo Working dir: $(pwd) + echo Files at this location: + ls -ltha + echo HOME: ${HOME} + echo LANG: ${LANG} SHELL: ${SHELL} + which python + echo LD_LIBRARY_PATH: ${LD_LIBRARY_PATH} + echo PYTHONPATH: \'${PYTHONPATH}\' + + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # ** here (it is expected that) ** + # pythonLocation: /opt/hostedtoolcache/Python/3.8.8/x64 + # LD_LIBRARY_PATH: /opt/hostedtoolcache/Python/3.8.8/x64/lib + # Working dir /home/runner/work/dpath-python/dpath-python + # HOME: /home/runner + # LANG: C.UTF-8 + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + - name: Install dependencies + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # requirements install the test framework, which is not + # required by the package in setup.py + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + run: | + python -m pip install --upgrade pip setuptools wheel + if [ -f requirements.txt ]; then + pip install -r requirements.txt; + fi + python setup.py install + echo which nose :$(which nose) + echo which nose2: $(which nose2) + echo which nose2-3.6: $(which nose2-3.6) + echo which nose2-3.8: $(which nose2-3.8) + + - name: Test + run: | + nose2 + diff --git a/dpath/segments.py b/dpath/segments.py index 65f8920..94ac471 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -2,7 +2,7 @@ from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound from dpath import options from fnmatch import fnmatchcase - +import sys def kvs(node): ''' @@ -22,7 +22,12 @@ def leaf(thing): leaf(thing) -> bool ''' - leaves = (bytes, str, int, float, bool, type(None)) + # resolve unicode issue in Python2.7, see test/test_unicode.py + # (TestEncoding.test_reproduce*) + if sys.version_info < (3, 0): + leaves = (bytes, str, unicode, int, float, bool, type(None)) + else: + leaves = (bytes, str, int, float, bool, type(None)) return isinstance(thing, leaves) diff --git a/dpath/util.py b/dpath/util.py index f90fd6e..d8ac0c7 100644 --- a/dpath/util.py +++ b/dpath/util.py @@ -1,5 +1,11 @@ -from collections.abc import MutableMapping -from collections.abc import MutableSequence +# deal with Python2.7 / Python3 difference +try: + from collections.abc import MutableMapping + from collections.abc import MutableSequence +except ImportError: + from collections import MutableMapping + from collections import MutableSequence + from dpath import options from dpath.exceptions import InvalidKeyName import dpath.segments diff --git a/tests/test_unicode.py b/tests/test_unicode.py index 104e108..38f8d1e 100644 --- a/tests/test_unicode.py +++ b/tests/test_unicode.py @@ -1,32 +1,109 @@ +# -*- coding: utf-8 -*- +# making this test autonomous and adding test for other Unicode issues +# found running under Python2.7 +import sys + +import unittest + import dpath.util +import dpath.segments as api + + +class TestEncoding(unittest.TestCase): + DO_DEBUG_PRINT = False + + def test_unicode_merge(self): + a = {'中': 'zhong'} + b = {'文': 'wen'} + + dpath.util.merge(a, b) + assert(len(a.keys()) == 2) + assert(a['中'] == 'zhong') + assert(a['文'] == 'wen') + + def test_unicode_search(self): + a = {'中': 'zhong'} + + results = [[x[0], x[1]] for x in dpath.util.search(a, '*', yielded=True)] + assert(len(results) == 1) + assert(results[0][0] == '中') + assert(results[0][1] == 'zhong') + + + def test_unicode_str_hybrid(self): + a = {'first': u'1'} + b = {u'second': '2'} + + dpath.util.merge(a, b) + assert(len(a.keys()) == 2) + assert(a[u'second'] == '2') + assert(a['second'] == u'2') + assert(a[u'first'] == '1') + assert(a['first'] == u'1') + + +# ...................................................................... +# Reproducing an issue in Python2.7, not in Python3, that boiled down to +# unicode support in api.leaf. This resulted in infinite loop in api.walk +# In following code: AA will be OK, before correction UU failed as shown below: +# +# Test of api.fold OK +# About to call api.fold with thing=()UU f=adder +# walk entered with obj=()UU, location=()() +# walk entered with obj=()U, location=()(0,) +# walk entered with obj=()U, location=()(0, 0) +# .... more deleted ... +# RuntimeError: maximum recursion depth exceeded while calling a Python object +# ...................................................................... + + def test_reproduce_issue(self): + + def f(o, p, a): + a[0] += 1 + for thing in ("AA", u"UU"): + if TestEncoding.DO_DEBUG_PRINT: + sys.stderr.write("About to call api.fold with thing=(%s)%s f=adder\n" + % (type(thing), thing)) + [count] = api.fold(thing, f, [0]) + assert count == len(tuple(api.walk(thing))) -def test_unicode_merge(): - a = {'中': 'zhong'} - b = {'文': 'wen'} + def test_reproduce_issue2(self): + for thing in ("AA", u"UU"): + if TestEncoding.DO_DEBUG_PRINT: + sys.stderr.write("About to call walk with arg=(%s)%s\n" + % (type(thing), thing)) + for pair in api.walk(thing): + sys.stderr.write("pair=%s\n" % repr(pair)) - dpath.util.merge(a, b) - assert(len(a.keys()) == 2) - assert(a['中'] == 'zhong') - assert(a['文'] == 'wen') + def test_reproduce_issue3(self): + for thing in ("AA", u"UU"): + if TestEncoding.DO_DEBUG_PRINT: + sys.stderr.write("About to call leaf and kvs with arg=(%s)%s\n" + % (type(thing), thing)) + sys.stderr.write("leaf(%s) => %s \n" % (thing, api.leaf(thing))) + sys.stderr.write("kvs(%s) => %s \n" % (thing, api.kvs(thing))) + assert api.leaf(thing) -def test_unicode_search(): - a = {'中': 'zhong'} +if __name__ == "__main__": + if "-h" in sys.argv: + description = """\ +This may run either under tox or standalone. When standalone +flags -h and -v are recognized, other flags are dealt with by unittest.main +and may select test cases. - results = [[x[0], x[1]] for x in dpath.util.search(a, '*', yielded=True)] - assert(len(results) == 1) - assert(results[0][0] == '中') - assert(results[0][1] == 'zhong') +Flags: + -h print this help and quit + -v print information messages on stderr +""" + print(description) + sys.exit(0) + if "-v" in sys.argv: + sys.argv = [x for x in sys.argv if x != "-v"] + TestEncoding.DO_DEBUG_PRINT = True + sys.stderr.write("Set verbose mode\n") -def test_unicode_str_hybrid(): - a = {'first': u'1'} - b = {u'second': '2'} + unittest.main() - dpath.util.merge(a, b) - assert(len(a.keys()) == 2) - assert(a[u'second'] == '2') - assert(a['second'] == u'2') - assert(a[u'first'] == '1') - assert(a['first'] == u'1') From 24027d0476e754a03febba27b32b00b895b01b7c Mon Sep 17 00:00:00 2001 From: Alain Lichnewsky Date: Thu, 22 Apr 2021 10:57:59 +0200 Subject: [PATCH 2/3] Correct YML error --- .github/workflows/python3Test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python3Test.yml b/.github/workflows/python3Test.yml index 929f4db..dd48666 100644 --- a/.github/workflows/python3Test.yml +++ b/.github/workflows/python3Test.yml @@ -22,7 +22,7 @@ jobs: - name: Set up Python 3.8 uses: actions/setup-python@v2 - with: + with: python-version: '3.8' architecture: 'x64' From 525a1b454d1ab93ac77ac1987df29731b80227ce Mon Sep 17 00:00:00 2001 From: Alain Lichnewsky Date: Thu, 22 Apr 2021 11:41:31 +0200 Subject: [PATCH 3/3] Corrected YAML error, now relying on yamllint ! --- .github/workflows/linterTest.yml | 14 ++-- .github/workflows/python3Test.yml | 106 +++++++++++++++--------------- 2 files changed, 60 insertions(+), 60 deletions(-) diff --git a/.github/workflows/linterTest.yml b/.github/workflows/linterTest.yml index 1f284eb..05ff1b8 100644 --- a/.github/workflows/linterTest.yml +++ b/.github/workflows/linterTest.yml @@ -8,22 +8,23 @@ name: LinterTest # including added functionality. # ------------------------------------------------------------ -# Controls when the action will run. +# Controls when the action will run. on: # - ## Not enabled, would triggers the workflow on push or pull request events but only - ## for the AL-addRegexp branch. + ## Not enabled, would triggers the workflow on push or pull request events but + ## only for the AL-addRegexp branch. #push: # branches: [ AL-addRegexp ] - + # Allows to run this workflow manually from the Github Actions tab workflow_dispatch: -# A workflow run is made up of one or more jobs that can run sequentially or in parallel +# A workflow run is made up of one or more jobs that can run sequentially or in +# parallel jobs: # This workflow contains a single job called "super-lint" super-lint: - # Steps represent a sequence of tasks that will be executed as part of the job + # Steps represent a sequence of tasks that will be executed by the job # Name the Job name: Lint code base # Set the type of machine to run on @@ -53,4 +54,3 @@ jobs: env: DEFAULT_BRANCH: AL-addRegexp GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - diff --git a/.github/workflows/python3Test.yml b/.github/workflows/python3Test.yml index dd48666..78927be 100644 --- a/.github/workflows/python3Test.yml +++ b/.github/workflows/python3Test.yml @@ -10,62 +10,62 @@ name: Test python package dpath-python on: #[push] workflow_dispatch: - + jobs: - test-python3: + test-python3: runs-on: ubuntu-latest - + steps: - - name: Checkout code - uses: actions/checkout@v2 - - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: '3.8' - architecture: 'x64' + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: '3.8' + architecture: 'x64' + + - name: Ascertain configuration + # + # Collect information concerning $HOME and the location of + # file(s) loaded from Github/ + run: | + echo Working dir: $(pwd) + echo Files at this location: + ls -ltha + echo HOME: ${HOME} + echo LANG: ${LANG} SHELL: ${SHELL} + which python + echo LD_LIBRARY_PATH: ${LD_LIBRARY_PATH} + echo PYTHONPATH: \'${PYTHONPATH}\' + + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # ** here (it is expected that) ** + # pythonLocation: /opt/hostedtoolcache/Python/3.8.8/x64 + # LD_LIBRARY_PATH: /opt/hostedtoolcache/Python/3.8.8/x64/lib + # Working dir /home/runner/work/dpath-python/dpath-python + # HOME: /home/runner + # LANG: C.UTF-8 + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + - name: Install dependencies - - name: Ascertain configuration - # - # Collect information concerning $HOME - # location of file loaded from Github/ - run: | - echo Working dir: $(pwd) - echo Files at this location: - ls -ltha - echo HOME: ${HOME} - echo LANG: ${LANG} SHELL: ${SHELL} - which python - echo LD_LIBRARY_PATH: ${LD_LIBRARY_PATH} - echo PYTHONPATH: \'${PYTHONPATH}\' - - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - # ** here (it is expected that) ** - # pythonLocation: /opt/hostedtoolcache/Python/3.8.8/x64 - # LD_LIBRARY_PATH: /opt/hostedtoolcache/Python/3.8.8/x64/lib - # Working dir /home/runner/work/dpath-python/dpath-python - # HOME: /home/runner - # LANG: C.UTF-8 - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - - - name: Install dependencies - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - # requirements install the test framework, which is not - # required by the package in setup.py - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - run: | - python -m pip install --upgrade pip setuptools wheel - if [ -f requirements.txt ]; then - pip install -r requirements.txt; - fi - python setup.py install - echo which nose :$(which nose) - echo which nose2: $(which nose2) - echo which nose2-3.6: $(which nose2-3.6) - echo which nose2-3.8: $(which nose2-3.8) + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # requirements install the test framework, which is not + # required by the package in setup.py + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + run: | + python -m pip install --upgrade pip setuptools wheel + if [ -f requirements.txt ]; then + pip install -r requirements.txt; + fi + python setup.py install + echo which nose :$(which nose) + echo which nose2: $(which nose2) + echo which nose2-3.6: $(which nose2-3.6) + echo which nose2-3.8: $(which nose2-3.8) - - name: Test - run: | - nose2 - + - name: Test + run: | + nose2