From b2609557b63b0681c9105b89a54fa4dfde75be61 Mon Sep 17 00:00:00 2001 From: Alain Lich Date: Sat, 8 Apr 2023 18:55:36 +0200 Subject: [PATCH 01/16] Adds support for re regular expressions in paths Restructured as requested, strictly required mods - flake8 compliant - nose2 tested - tox tested for Python3.11 and Pypy3.9 - improved some tests (require verification of result...) - documentation README.rst synchronized Still TBD: - check of .github/workflows Test environment: - Running on ARM-64 Linux version 5.15.49-linuxkit (root@buildkitsandbox) (gcc (Alpine 10.2.1_pre1) 10.2.1 20201203, GNU ld (GNU Binutils) 2.35.2) #1 SMP PREEMPT Tue Sep 13 07:51:32 UTC 2022 - Python 3.11.2 - pypy3 --version : Python 3.9.16 (7.3.11+dfsg-2, Feb 06 2023, 17:14:22) [PyPy 7.3.11 with GCC 12.2.0] --- README.rst | 79 ++++++++++ dpath/__init__.py | 21 ++- dpath/options.py | 6 + dpath/segments.py | 21 ++- tests/test_various_exts.py | 308 +++++++++++++++++++++++++++++++++++++ tox.ini | 6 +- 6 files changed, 431 insertions(+), 10 deletions(-) create mode 100644 tests/test_various_exts.py diff --git a/README.rst b/README.rst index 0ad3ad2..7f6edf6 100644 --- a/README.rst +++ b/README.rst @@ -111,6 +111,9 @@ elements in ``x['a']['b']`` where the key is equal to the glob ``'[cd]'``. Okay. } } +**Note** : Using Python's `re` regular expressions instead of globs is explained +below re_regexp_. + ... Wow that was easy. What if I want to iterate over the results, and not get a merged view? @@ -438,6 +441,82 @@ To get around this, you can sidestep the whole "filesystem path" style, and aban >>> dpath.get(['a', 'b/c']) 0 +.. _re_regexp: + +Globs too imprecise? Use Python's `re` Regular Expressions +========================================================== + +Python's `re` regular expressions PythonRe_ may be used as follows: + + .. _PythonRe: https://docs.python.org/3/library/re.html + + - This facility is enabled by default, but may be disabled (for backwards + compatibility in the unlikely cases where a path expression component would start + with '{' and end in '}'): + + .. code-block:: python + + >>> import dpath + >>> # disable + >>> dpath.options.DPATH_ACCEPT_RE_REGEXP = False + >>> # enable + >>> dpath.options.DPATH_ACCEPT_RE_REGEXP = True + + - Now a path component may also be specified : + + - in a path expression, as {} where `` is a regular expression + accepted by the standard Python module `re`. For example: + + .. code-block:: python + + >>> selPath = 'Config/{(Env|Cmd)}' + >>> x = dpath.util.search(js.lod, selPath) + + .. code-block:: python + + >>> selPath = '{(Config|Graph)}/{(Env|Cmd|Data)}' + >>> x = dpath.util.search(js.lod, selPath) + + - When using the list form for a path, a list element can also + be expressed as + + - a string as above + - the output of :: `re.compile( args )`` + + An example: + + .. code-block:: python + + >>> selPath = [ re.compile('(Config|Graph)') , re.compile('(Env|Cmd|Data)') ] + >>> x = dpath.util.search(js.lod, selPath) + + More examples from a realistic json context: + + +-----------------------------------------+--------------------------------------+ + + **Extended path glob** | **Designates** + + +-----------------------------------------+--------------------------------------+ + + "\*\*/{[^A-Za-z]{2}$}" | "Id" + + +-----------------------------------------+--------------------------------------+ + + r"\*/{[A-Z][A-Za-z\\d]*$}" | "Name","Id","Created", "Scope",... + + +-----------------------------------------+--------------------------------------+ + + r"\*\*/{[A-Z][A-Za-z\\d]*\d$}" | EnableIPv6" + + +-----------------------------------------+--------------------------------------+ + + r"\*\*/{[A-Z][A-Za-z\\d]*Address$}" | "Containers/199c5/MacAddress" + + +-----------------------------------------+--------------------------------------+ + + With Python's character string conventions, required backslashes in the `re` syntax + can be entered either in raw strings or using double backslashes, thus + the following are equivalent: + + +-----------------------------------------+----------------------------------------+ + + *with raw strings* | *equivalent* with double backslash + + +-----------------------------------------+----------------------------------------+ + + r"\*\*/{[A-Z][A-Za-z\\d]*\\d$}" | "\*\*/{[A-Z][A-Za-z\\\\d]*\\\\d$}" + + +-----------------------------------------+----------------------------------------+ + + r"\*\*/{[A-Z][A-Za-z\\d]*Address$}" | "\*\*/{[A-Z][A-Za-z\\\\d]*Address$}" + + +-----------------------------------------+----------------------------------------+ + + dpath.segments : The Low-Level Backend ====================================== diff --git a/dpath/__init__.py b/dpath/__init__.py index 9f56e6b..c907974 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -27,6 +27,10 @@ from dpath.exceptions import InvalidKeyName, PathNotFound from dpath.types import MergeType, PathSegment, Creator, Filter, Glob, Path, Hints +import sys +import re + + _DEFAULT_SENTINEL = object() @@ -45,7 +49,22 @@ def _split_path(path: Path, separator: Optional[str] = "/") -> Union[List[PathSe else: split_segments = path.lstrip(separator).split(separator) - return split_segments + final = [] + for segment in split_segments: + if (options.DPATH_ACCEPT_RE_REGEXP and isinstance(segment, str) + and segment[0] == '{' and segment[-1] == '}'): + try: + rs = segment[1:-1] + rex = re.compile(rs) + except Exception as reErr: + print(f"Error in segment '{segment}' string '{rs}' not accepted" + + f"as re.regexp:\n\t{reErr}", + file=sys.stderr) + raise reErr + final.append(rex) + else: + final.append(segment) + return final def new(obj: MutableMapping, path: Path, value, separator="/", creator: Creator = None) -> MutableMapping: diff --git a/dpath/options.py b/dpath/options.py index 41f35c4..260d922 100644 --- a/dpath/options.py +++ b/dpath/options.py @@ -1 +1,7 @@ ALLOW_EMPTY_STRING_KEYS = False + +# Extension to interpret path segments "{rrr}" as re.regexp "rrr" enabled by default. +# Disable to preserve backwards compatibility in the case where a user has a +# path "a/b/{cd}" where the brackets are intentional and do not denote a request +# to re.compile cd +DPATH_ACCEPT_RE_REGEXP = True diff --git a/dpath/segments.py b/dpath/segments.py index c3c9846..46deeee 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -6,6 +6,8 @@ from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound from dpath.types import PathSegment, Creator, Hints, Glob, Path, SymmetricInt +from re import Pattern + def make_walkable(node) -> Iterator[Tuple[PathSegment, Any]]: """ @@ -182,9 +184,11 @@ def match(segments: Path, glob: Glob): or more star segments and the type will be coerced to match that of the segment. - A segment is considered to match a glob if the function - fnmatch.fnmatchcase returns True. If fnmatchcase returns False or - throws an exception the result will be False. + A segment is considered to match a glob when either: + - the segment is a String : the function fnmatch.fnmatchcase returns True. + If fnmatchcase returns False or throws an exception the result will be False. + - or, the segment is a re.Pattern (result of re.compile) and re.Pattern.match returns + a match match(segments, glob) -> bool """ @@ -241,10 +245,13 @@ def match(segments: Path, glob: Glob): s = str(s) try: - # Let's see if the glob matches. We will turn any kind of - # exception while attempting to match into a False for the - # match. - if not fnmatchcase(s, g): + # Let's see if the glob or the regular expression matches. We will turn any kind of + # exception while attempting to match into a False for the match. + if isinstance(g, Pattern): + mobj = g.match(s) + if mobj is None: + return False + elif not fnmatchcase(s, g): return False except: return False diff --git a/tests/test_various_exts.py b/tests/test_various_exts.py new file mode 100644 index 0000000..48b75b7 --- /dev/null +++ b/tests/test_various_exts.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# -*- mode: Python -*- +# +# (C) Alain Lichnewsky, 2022 +# +# Test support of extended specs with re.regex in many functionalities that use path +# specifications. +# +import sys +import re + +from copy import copy + +import unittest +import dpath as DP + +# check that how the options have been set +print(f"At entry in test_path_ext DPATH_ACCEPT_RE_REGEXP = {DP.options.DPATH_ACCEPT_RE_REGEXP}", file=sys.stderr) + +if not DP.options.DPATH_ACCEPT_RE_REGEXP: + print("This test doesn't make sense with DPATH_ACCEPT_RE_REGEXP = False", file=sys.stderr) + DP.options.DPATH_ACCEPT_RE_REGEXP = True # enable re.regexp support in path expr. + + +# one class per function to be tested +class SampleDicts(): + + def build(self): + self.d1 = { + "a001": { + "b2": { + "c1.2": { + "d.dd": 0, + "e.ee": 1, + "f.f0": 2, + }, + }, + }, + } + + self.specs1 = (([re.compile(".*")], "a001", True), + ([re.compile("[a-z]+$")], "a001", False), + (["*", re.compile(".*")], "a001", False), + (["*", "*", re.compile(".*")], "a001", False), + (["*", re.compile("[a-z]+\\d+$")], "a001", False), + (["*", re.compile("[a-z]+[.][a-z]+$")], "a001", False), + (["**", re.compile(".*")], "a001", True), + (["**", re.compile("[a-z]+\\d+$")], "a001", True), + (["**", re.compile("[a-z]+[.][a-z]+$")], "a001", False) + ) + + self.specs1Pairs = (([re.compile(".*")], ("a001",)), + ([re.compile("[a-z]+$")], None), + (["*", re.compile(".*")], ("a001/b2",)), + (["*", "*", re.compile(".*")], ("a001/b2/c1.2",)), + (["*", re.compile("[a-z]+\\d+$")], ("a001/b2",)), + (["*", re.compile("[a-z]+[.][a-z]+$")], None), + (["**", re.compile(".*")], + ("a001", "a001/b2", "a001/b2/c1.2", "a001/b2/c1.2/d.dd", + "a001/b2/c1.2/e.ee", "a001/b2/c1.2/f.f0")), + (["**", re.compile("[a-z]+\\d+$")], ("a001", "a001/b2")), + (["**", re.compile("[a-z]+[.][a-z]+$")], + ("a001/b2/c1.2/d.dd", "a001/b2/c1.2/e.ee")) + ) + + self.specs1GetPairs = (([re.compile(".*")], {'b2': {'c1.2': {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}}}), + ([re.compile("[a-z]+$")], ('*NONE*',)), + (["*", re.compile(".*")], {'c1.2': {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}}), + (["*", "*", re.compile(".*")], {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}), + (["*", re.compile("[a-z]+\\d+$")], {'c1.2': {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}}), + (["*", re.compile("[a-z]+[.][a-z]+$")], ('*NONE*',)), + (["**", re.compile(".*")], "*FAIL*"), + (["**", re.compile("[a-z]+\\d+$")], "*FAIL*"), + (["**", re.compile("[a-z]+[.][a-z]+$")], "*FAIL*"), + ) + + self.d2 = {"Name": "bridge", + "Id": "333d22b3724", + "Created": "2022-12-08T09:02:33.360812052+01:00", + "Scope": "local", + "Driver": "bridge", + "EnableIPv6": False, + "IPAM": { + "Driver": "default", + "Options": None, + "Config": + { + "Subnet": "172.17.0.0/16", + "Gateway": "172.17.0.1" + } + }, + "Internal": False, + "Attachable": False, + "Ingress": False, + "ConfigFrom": { + "Network": "" + }, + "ConfigOnly": False, + "Containers": { + "199c590e8f13477": { + "Name": "al_dpath", + "EndpointID": "3042bbe16160a63b7", + "MacAddress": "02:34:0a:11:10:22", + "IPv4Address": "172.17.0.2/16", + "IPv6Address": "" + } + }, + "Options": { + "com.docker.network.bridge.default_bridge": "true", + "com.docker.network.bridge.enable_icc": "true", + "com.docker.network.bridge.enable_ip_masquerade": "true", + "com.docker.network.bridge.host_binding_ipv4": "0.0.0.0", + "com.docker.network.bridge.name": "docker0", + "com.docker.network.driver.mtu": "1500" + }, + "Labels": {} + } + + self.specs2Pairs = ((["*", re.compile("[A-Z][a-z\\d]*$")], + ("IPAM/Driver", "IPAM/Options", "IPAM/Config", "ConfigFrom/Network")), + (["**", re.compile("[A-Z][a-z\\d]*$")], + ("Name", "Id", "Created", "Scope", "Driver", "Internal", "Attachable", + "Ingress", "Containers", "Options", "Labels", "IPAM/Driver", "IPAM/Options", + "IPAM/Config", "IPAM/Config/Subnet", "IPAM/Config/Gateway", + "ConfigFrom/Network", "Containers/199c590e8f13477/Name", + "Containers/199c590e8f13477/MacAddress")), + (["**", re.compile("[A-Z][A-Za-z\\d]*Address$")], + ("Containers/199c590e8f13477/MacAddress", "Containers/199c590e8f13477/IPv4Address", + "Containers/199c590e8f13477/IPv6Address")), + (["**", re.compile("[A-Za-z]+\\d+$")], ("EnableIPv6",)), + (["**", re.compile("\\d+[.]\\d+")], None), + + # repeated intentionally using raw strings rather than '\\' escapes + + (["*", re.compile(r"[A-Z][a-z\d]*$")], + ("IPAM/Driver", "IPAM/Options", "IPAM/Config", "ConfigFrom/Network")), + (["**", re.compile(r"[A-Z][a-z\d]*$")], + ("Name", "Id", "Created", "Scope", "Driver", "Internal", "Attachable", + "Ingress", "Containers", "Options", "Labels", "IPAM/Driver", "IPAM/Options", + "IPAM/Config", "IPAM/Config/Subnet", "IPAM/Config/Gateway", + "ConfigFrom/Network", "Containers/199c590e8f13477/Name", + "Containers/199c590e8f13477/MacAddress")), + (["**", re.compile(r"[A-Z][A-Za-z\d]*Address$")], + ("Containers/199c590e8f13477/MacAddress", "Containers/199c590e8f13477/IPv4Address", + "Containers/199c590e8f13477/IPv6Address")), + (["**", re.compile(r"[A-Za-z]+\d+$")], ("EnableIPv6",)), + (["**", re.compile(r"\d+[.]\d+")], None) + ) + + self.specs3Pairs = (("**/{[^A-Za-z]{2}$}", ("Id",)), + ("*/{[A-Z][A-Za-z\\d]*$}", ("Name", "Id", "Created", "Scope", "Driver", "Internal", + "Attachable", "Ingress", "Containers", "Options", "Labels", + "IPAM/Driver", "IPAM/Options", "IPAM/Config", "IPAM/Config/Subnet", + "IPAM/Config/Gateway", "ConfigFrom/Network", "Containers/199c590e8f13477/Name", + "Containers/199c590e8f13477/MacAddress")), + ("**/{[A-Z][A-Za-z\\d]*\\d$}", ("EnableIPv6",)), + ("**/{[A-Z][A-Za-z\\d]*Address$}", ("Containers/199c590e8f13477/MacAddress", + "Containers/199c590e8f13477/IPv4Address", + "Containers/199c590e8f13477/IPv6Address")), + + # repeated intentionally using raw strings rather than '\\' escapes + + (r"**/{[A-Z][A-Za-z\d]*\d$}", ("EnableIPv6",)), + (r"**/{[A-Z][A-Za-z\d]*Address$}", ("Containers/199c590e8f13477/MacAddress", + "Containers/199c590e8f13477/IPv4Address", + "Containers/199c590e8f13477/IPv6Address")), + ) + return self + + +class TestSearch(unittest.TestCase): + + def test1(self): + print("Entered test1", file=sys.__stderr__) + dicts = SampleDicts().build() + dict1 = dicts.d1 + specs = dicts.specs1Pairs + for (spec, expect) in specs: + print(f"Spec={spec}", file=sys.stderr) + for (path, value) in DP.search(dict1, spec, yielded=True): + print(f"\tpath={path}\tv={value}\n\texpected={expect}", + file=sys.stderr) + if path is None: + assert expect is None + else: + assert (path in expect) + print("\n", file=sys.stderr) + + def test2(self): + print("Entered test2", file=sys.__stderr__) + + def afilter(x): + # print(f"In afilter x = {x}({type(x)})", file=sys.stderr) + if isinstance(x, int): + return True + return False + + dicts = SampleDicts().build() + dict1 = dicts.d1 + specs = dicts.specs1 + for spec in (s[0] for s in specs): + print(f"Spec={spec}", file=sys.stderr) + for ret in DP.search(dict1, spec, yielded=True, afilter=afilter): + print(f"\tret={ret}", file=sys.stderr) + assert (isinstance(ret[1], int)) + + def test3(self): + print("Entered test3", file=sys.__stderr__) + dicts = SampleDicts().build() + dict1 = dicts.d2 + specs = dicts.specs2Pairs + for (spec, expect) in specs: + print(f"Spec={spec}", file=sys.stderr) + for (path, value) in DP.search(dict1, spec, yielded=True): + print(f"\tpath={path}\tv={value}", file=sys.stderr) + if path is None: + assert expect is None + else: + assert (path in expect) + + def test4(self): + print("Entered test4", file=sys.__stderr__) + dicts = SampleDicts().build() + dict1 = dicts.d2 + specs = dicts.specs3Pairs + for (spec, expect) in specs: + print(f"Spec={spec}", file=sys.stderr) + for (path, value) in DP.search(dict1, spec, yielded=True): + print(f"\tpath={path}\tv={value}", file=sys.stderr) + if path is None: + assert expect is None + else: + assert (path in expect) + + +class TestGet(unittest.TestCase): + + def test1(self): + print("Entered test1", file=sys.__stderr__) + + dicts = SampleDicts().build() + dict1 = dicts.d1 + specs = dicts.specs1GetPairs + for (spec, expect) in specs: + print(f"Spec={spec}", file=sys.stderr) + try: + ret = DP.get(dict1, spec, default=("*NONE*",)) + print(f"\tret={ret}", file=sys.stderr) + assert (ret == expect) + except Exception as err: + print("\t get fails:", err, type(err), file=sys.stderr) + assert (expect == "*FAIL*") + + +class TestDelete(unittest.TestCase): + def test1(self): + print("This is test1", file=sys.stderr) + dict1 = { + "a": { + "b": 0, + "12": 0, + }, + "a0": { + "b": 0, + }, + } + + specs = (re.compile("[a-z]+$"), re.compile("[a-z]+\\d+$"), + "{[a-z]+\\d+$}") + i = 0 + for spec in specs: + dict = copy(dict1) + print(f"spec={spec}") + print(f"Before deletion dict={dict}", file=sys.stderr) + DP.delete(dict, [spec]) + print(f"After deletion dict={dict}", file=sys.stderr) + if i == 0: + assert (dict == {"a0": {"b": 0, }, }) + else: + assert (dict == {"a": {"b": 0, "12": 0, }}) + i += 1 + + +class TestView(unittest.TestCase): + def test1(self): + print("Entered test1", file=sys.__stderr__) + + dicts = SampleDicts().build() + dict1 = dicts.d1 + specs = dicts.specs1 + for spec in specs: + for ret in DP.segments.view(dict1, spec[0]): + print(f"\tview {spec=} returns:{ret}", file=sys.stderr) + assert ret == spec[1] + + +class TestMatch(unittest.TestCase): + def test1(self): + print("Entered test1", file=sys.__stderr__) + + dicts = SampleDicts().build() + dict1 = dicts.d1 + specs = dicts.specs1 + for spec in specs: + ret = DP.segments.match(dict1, spec[0]) + print(f"\tmatch {spec=} returns:{ret}", file=sys.stderr) + assert ret == spec[2] diff --git a/tox.ini b/tox.ini index d613837..abe73a8 100644 --- a/tox.ini +++ b/tox.ini @@ -4,17 +4,19 @@ # and then run "tox" from this directory. [flake8] -ignore = E501,E722 +ignore = E501,E722,W503 [tox] -envlist = pypy37, py38, py39, py310 +envlist = pypy37, pypy39, py38, py39, py310, py311 [gh-actions] python = pypy-3.7: pypy37 + pypy-3.9: pypy39 3.8: py38 3.9: py39 3.10: py310 + 3.11: py311 [testenv] deps = From 8a468c1bfc6dbee0a7c3b86ae5a8ad4535e51682 Mon Sep 17 00:00:00 2001 From: Alain Lich Date: Sun, 9 Apr 2023 11:31:53 +0200 Subject: [PATCH 02/16] Add a workflow which is launched manually within selectable branch - new file: .github/workflows/python3Test.yml - remove this for PR --- .github/workflows/python3Test.yml | 95 +++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 .github/workflows/python3Test.yml diff --git a/.github/workflows/python3Test.yml b/.github/workflows/python3Test.yml new file mode 100644 index 0000000..404a445 --- /dev/null +++ b/.github/workflows/python3Test.yml @@ -0,0 +1,95 @@ +name: Test python package dpath-python with regexp extension + # ------------------------------------------------------------ + # (C) Alain Lichnewsky, 2021, 2022, 2023 + # + # For running under Github's Actions + # + # Script performs basic test of the Python-3 version of the package + # including added functionality (regexp in search paths). + # ------------------------------------------------------------ + + # ***************************** + # ADDED FOR TESTING PRIOR TO PR + # REMOVE FROM PR submission + # ***************************** + +on: + workflow_dispatch: + # Allows manual dispatch from the Actions tab + +jobs: + test-python3: + + timeout-minutes: 60 + + runs-on: ubuntu-latest + + strategy: + matrix: + # Match versions specified in tox.ini + python-version: ['3.8', '3.9', '3.10','3.11', 'pypy3.7', 'pypy3.9'] + + steps: + - name: Checkout code + uses: actions/checkout@main + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@main + with: + python-version: ${{ matrix.python-version }} + architecture: 'x64' + + - name: Ascertain configuration + # + # Collect information concerning $HOME and the location of + # file(s) loaded from Github/ + run: | + echo Working dir: $(pwd) + echo Files at this location: + ls -ltha + echo HOME: ${HOME} + echo LANG: ${LANG} SHELL: ${SHELL} + which python + echo LD_LIBRARY_PATH: ${LD_LIBRARY_PATH} + echo PYTHONPATH: \'${PYTHONPATH}\' + + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # ** here (it is expected that) ** + # pythonLocation: /opt/hostedtoolcache/Python/3.8.8/x64 + # LD_LIBRARY_PATH: /opt/hostedtoolcache/Python/3.8.8/x64/lib + # Working dir /home/runner/work/dpath-python/dpath-python + # HOME: /home/runner + # LANG: C.UTF-8 + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + - name: Install dependencies + shell: bash + if: always() + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # requirements install the test framework, which is not + # required by the package in setup.py + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + run: | + python -m pip install --upgrade pip setuptools wheel \ + nose2 hypothesis + if [ -f requirements.txt ]; then + pip install -r requirements.txt; + fi + python setup.py install + echo which nose :$(which nose) + echo which nose2: $(which nose2) + echo which nose2-3.6: $(which nose2-3.6) + echo which nose2-3.8: $(which nose2-3.8) + + - name: Tox testing + shell: bash + if: always() + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # move to tox testing, otherwise will have to parametrize + # nose in more details; here tox.ini will apply + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + run: | + pip install tox + echo "Installed tox" + tox + echo "Ran tox" From b5827d3e2551f3ba4d24a441524246f8f85e87ae Mon Sep 17 00:00:00 2001 From: Alain Lich Date: Sun, 9 Apr 2023 18:35:32 +0200 Subject: [PATCH 03/16] Corrected tests/test_various_exts.py pypy-3.7 incompat. - f-strings support = for self-documenting expression introduced with Python 3.8 --- tests/test_various_exts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_various_exts.py b/tests/test_various_exts.py index 48b75b7..ac36779 100644 --- a/tests/test_various_exts.py +++ b/tests/test_various_exts.py @@ -291,7 +291,7 @@ def test1(self): specs = dicts.specs1 for spec in specs: for ret in DP.segments.view(dict1, spec[0]): - print(f"\tview {spec=} returns:{ret}", file=sys.stderr) + print(f"\tview spec:{spec} returns:{ret}", file=sys.stderr) assert ret == spec[1] @@ -304,5 +304,5 @@ def test1(self): specs = dicts.specs1 for spec in specs: ret = DP.segments.match(dict1, spec[0]) - print(f"\tmatch {spec=} returns:{ret}", file=sys.stderr) + print(f"\tmatch spec:{spec} returns:{ret}", file=sys.stderr) assert ret == spec[2] From bccb6d053ffefe3d8ddbc7d1a92d4d80df51546f Mon Sep 17 00:00:00 2001 From: Alain Lich Date: Sun, 9 Apr 2023 20:50:03 +0200 Subject: [PATCH 04/16] Adds support for re regular expressions in paths - includes only required changes for feature, as suggested in @moomoohk comment on April 5th - test improvements - tested on python3.8,...,3.11 and pypy3.7 & 3.9 --- .github/workflows/python3Test.yml | 95 ------------------------------- 1 file changed, 95 deletions(-) delete mode 100644 .github/workflows/python3Test.yml diff --git a/.github/workflows/python3Test.yml b/.github/workflows/python3Test.yml deleted file mode 100644 index 404a445..0000000 --- a/.github/workflows/python3Test.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Test python package dpath-python with regexp extension - # ------------------------------------------------------------ - # (C) Alain Lichnewsky, 2021, 2022, 2023 - # - # For running under Github's Actions - # - # Script performs basic test of the Python-3 version of the package - # including added functionality (regexp in search paths). - # ------------------------------------------------------------ - - # ***************************** - # ADDED FOR TESTING PRIOR TO PR - # REMOVE FROM PR submission - # ***************************** - -on: - workflow_dispatch: - # Allows manual dispatch from the Actions tab - -jobs: - test-python3: - - timeout-minutes: 60 - - runs-on: ubuntu-latest - - strategy: - matrix: - # Match versions specified in tox.ini - python-version: ['3.8', '3.9', '3.10','3.11', 'pypy3.7', 'pypy3.9'] - - steps: - - name: Checkout code - uses: actions/checkout@main - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@main - with: - python-version: ${{ matrix.python-version }} - architecture: 'x64' - - - name: Ascertain configuration - # - # Collect information concerning $HOME and the location of - # file(s) loaded from Github/ - run: | - echo Working dir: $(pwd) - echo Files at this location: - ls -ltha - echo HOME: ${HOME} - echo LANG: ${LANG} SHELL: ${SHELL} - which python - echo LD_LIBRARY_PATH: ${LD_LIBRARY_PATH} - echo PYTHONPATH: \'${PYTHONPATH}\' - - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - # ** here (it is expected that) ** - # pythonLocation: /opt/hostedtoolcache/Python/3.8.8/x64 - # LD_LIBRARY_PATH: /opt/hostedtoolcache/Python/3.8.8/x64/lib - # Working dir /home/runner/work/dpath-python/dpath-python - # HOME: /home/runner - # LANG: C.UTF-8 - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - - - name: Install dependencies - shell: bash - if: always() - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - # requirements install the test framework, which is not - # required by the package in setup.py - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - run: | - python -m pip install --upgrade pip setuptools wheel \ - nose2 hypothesis - if [ -f requirements.txt ]; then - pip install -r requirements.txt; - fi - python setup.py install - echo which nose :$(which nose) - echo which nose2: $(which nose2) - echo which nose2-3.6: $(which nose2-3.6) - echo which nose2-3.8: $(which nose2-3.8) - - - name: Tox testing - shell: bash - if: always() - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - # move to tox testing, otherwise will have to parametrize - # nose in more details; here tox.ini will apply - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - run: | - pip install tox - echo "Installed tox" - tox - echo "Ran tox" From bf42f9bdddc42bf771a8d37af1b1271b3737602b Mon Sep 17 00:00:00 2001 From: Alain Lich Date: Wed, 12 Apr 2023 10:25:22 +0200 Subject: [PATCH 05/16] Corrected documentation --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 7f6edf6..bd2bdb5 100644 --- a/README.rst +++ b/README.rst @@ -470,12 +470,12 @@ Python's `re` regular expressions PythonRe_ may be used as follows: .. code-block:: python >>> selPath = 'Config/{(Env|Cmd)}' - >>> x = dpath.util.search(js.lod, selPath) + >>> x = dpath.search(js.lod, selPath) .. code-block:: python >>> selPath = '{(Config|Graph)}/{(Env|Cmd|Data)}' - >>> x = dpath.util.search(js.lod, selPath) + >>> x = dpath.search(js.lod, selPath) - When using the list form for a path, a list element can also be expressed as @@ -488,7 +488,7 @@ Python's `re` regular expressions PythonRe_ may be used as follows: .. code-block:: python >>> selPath = [ re.compile('(Config|Graph)') , re.compile('(Env|Cmd|Data)') ] - >>> x = dpath.util.search(js.lod, selPath) + >>> x = dpath.search(js.lod, selPath) More examples from a realistic json context: From 9ae59256e8403887fe8343021a006156e827d97b Mon Sep 17 00:00:00 2001 From: Alain Lich Date: Sat, 15 Apr 2023 15:00:22 +0200 Subject: [PATCH 06/16] Implemented changes suggested by maintainer - types: addition of GlobExtend where both a glob and a regex are allowed - code : __init__.py: improved _split_path - added InvalidRegex exception - test improvements (assert expected result) and additions - change default behaviour DPATH_ACCEPT_RE_REGEXP_IN_STRING=False, kept documentation current Flake8 compliant, tested Python11/Pypy9 on debian/ARM64 Also added a workflow for testing in arbitrary branch, expected to be suppressed in PR, but more testing is in order at this time!! --- .github/workflows/python3Test.yml | 95 +++++++++++++++++++++++ README.rst | 14 ++-- dpath/__init__.py | 61 +++++++++------ dpath/exceptions.py | 5 ++ dpath/options.py | 5 +- dpath/segments.py | 5 +- dpath/types.py | 14 ++++ tests/test_various_exts.py | 125 +++++++++++++++++++++--------- 8 files changed, 252 insertions(+), 72 deletions(-) create mode 100644 .github/workflows/python3Test.yml diff --git a/.github/workflows/python3Test.yml b/.github/workflows/python3Test.yml new file mode 100644 index 0000000..404a445 --- /dev/null +++ b/.github/workflows/python3Test.yml @@ -0,0 +1,95 @@ +name: Test python package dpath-python with regexp extension + # ------------------------------------------------------------ + # (C) Alain Lichnewsky, 2021, 2022, 2023 + # + # For running under Github's Actions + # + # Script performs basic test of the Python-3 version of the package + # including added functionality (regexp in search paths). + # ------------------------------------------------------------ + + # ***************************** + # ADDED FOR TESTING PRIOR TO PR + # REMOVE FROM PR submission + # ***************************** + +on: + workflow_dispatch: + # Allows manual dispatch from the Actions tab + +jobs: + test-python3: + + timeout-minutes: 60 + + runs-on: ubuntu-latest + + strategy: + matrix: + # Match versions specified in tox.ini + python-version: ['3.8', '3.9', '3.10','3.11', 'pypy3.7', 'pypy3.9'] + + steps: + - name: Checkout code + uses: actions/checkout@main + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@main + with: + python-version: ${{ matrix.python-version }} + architecture: 'x64' + + - name: Ascertain configuration + # + # Collect information concerning $HOME and the location of + # file(s) loaded from Github/ + run: | + echo Working dir: $(pwd) + echo Files at this location: + ls -ltha + echo HOME: ${HOME} + echo LANG: ${LANG} SHELL: ${SHELL} + which python + echo LD_LIBRARY_PATH: ${LD_LIBRARY_PATH} + echo PYTHONPATH: \'${PYTHONPATH}\' + + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # ** here (it is expected that) ** + # pythonLocation: /opt/hostedtoolcache/Python/3.8.8/x64 + # LD_LIBRARY_PATH: /opt/hostedtoolcache/Python/3.8.8/x64/lib + # Working dir /home/runner/work/dpath-python/dpath-python + # HOME: /home/runner + # LANG: C.UTF-8 + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + - name: Install dependencies + shell: bash + if: always() + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # requirements install the test framework, which is not + # required by the package in setup.py + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + run: | + python -m pip install --upgrade pip setuptools wheel \ + nose2 hypothesis + if [ -f requirements.txt ]; then + pip install -r requirements.txt; + fi + python setup.py install + echo which nose :$(which nose) + echo which nose2: $(which nose2) + echo which nose2-3.6: $(which nose2-3.6) + echo which nose2-3.8: $(which nose2-3.8) + + - name: Tox testing + shell: bash + if: always() + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # move to tox testing, otherwise will have to parametrize + # nose in more details; here tox.ini will apply + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + run: | + pip install tox + echo "Installed tox" + tox + echo "Ran tox" diff --git a/README.rst b/README.rst index bd2bdb5..fbfdc22 100644 --- a/README.rst +++ b/README.rst @@ -450,17 +450,19 @@ Python's `re` regular expressions PythonRe_ may be used as follows: .. _PythonRe: https://docs.python.org/3/library/re.html - - This facility is enabled by default, but may be disabled (for backwards - compatibility in the unlikely cases where a path expression component would start - with '{' and end in '}'): + - The recognition of such regular expressions in strings is disabled by default, but may be easily + enabled ( Set up this way for backwards compatibility in the cases where a path + expression component would start with '{' and end in '}'). + - Irrespective of this setting, the user can use `re` regular expressions in the list form of + paths (see below). .. code-block:: python >>> import dpath - >>> # disable - >>> dpath.options.DPATH_ACCEPT_RE_REGEXP = False >>> # enable - >>> dpath.options.DPATH_ACCEPT_RE_REGEXP = True + >>> dpath.options.DPATH_ACCEPT_RE_REGEXP_IN_STRING = True + >>> # disable + >>> dpath.options.DPATH_ACCEPT_RE_REGEXP_IN_STRING = False - Now a path component may also be specified : diff --git a/dpath/__init__.py b/dpath/__init__.py index c907974..e15f03a 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -22,49 +22,60 @@ from collections.abc import MutableMapping, MutableSequence from typing import Union, List, Any, Callable, Optional +import re from dpath import segments, options -from dpath.exceptions import InvalidKeyName, PathNotFound -from dpath.types import MergeType, PathSegment, Creator, Filter, Glob, Path, Hints - -import sys -import re +from dpath.exceptions import InvalidKeyName, PathNotFound, InvalidRegex +from dpath.types import MergeType, PathSegment, Creator, Filter, Glob, GlobExtend, Path, Hints _DEFAULT_SENTINEL = object() -def _split_path(path: Path, separator: Optional[str] = "/") -> Union[List[PathSegment], PathSegment]: +def _split_path(path: GlobExtend, separator: Optional[str] = "/") -> Union[List[PathSegment], PathSegment]: """ - Given a path and separator, return a tuple of segments. If path is - already a non-leaf thing, return it. + Given a path and separator, return a tuple of segments; string segments that represent + (i.e. "{.*}") regexp are re.compile'd. + + If path is already a non-leaf thing, return it: this covers sequences of strings + and re.Patterns. Note that a string path with the separator at index[0] will have the separator stripped off. If you pass a list path, the separator is ignored, and is assumed to be part of each key glob. It will not be - stripped. + stripped (i.e. a first list element can be an empty string). + + Errors in re.compilation raise InvalidRegex exception. """ + # First split the path into segments, validate wrt. type annotation GlobExtend if not segments.leaf(path): split_segments = path + elif isinstance(path, re.Pattern): + # Allow a path which is a single re.Pattern + split_segments = (path,) + elif isinstance(path, (int, float, bool, type(None))): + # This protects against situations that are not screened by segment.leaf. These should + # not occur with spec. "path:GlobExtend", but type annotations do not check arguments. + # Thus the error message than is clearer than "...object has no attribute lstrip" + raise ValueError(f"Error: scalars cannot appear outside of sequence in {path}") else: split_segments = path.lstrip(separator).split(separator) - final = [] + # now we re.compile segments that represent re.Regexps. + split_compiled_segments = [] for segment in split_segments: - if (options.DPATH_ACCEPT_RE_REGEXP and isinstance(segment, str) - and segment[0] == '{' and segment[-1] == '}'): + if options.DPATH_ACCEPT_RE_REGEXP_IN_STRING and isinstance(segment, str) and segment[0] == "{" and segment[-1] == "}": try: rs = segment[1:-1] rex = re.compile(rs) - except Exception as reErr: - print(f"Error in segment '{segment}' string '{rs}' not accepted" - + f"as re.regexp:\n\t{reErr}", - file=sys.stderr) - raise reErr - final.append(rex) + except re.error as reErr: + raise InvalidRegex(f"In segment '{segment}' string '{rs}' not accepted" + + f" as re.regexp:\n==>\t{reErr}") + split_compiled_segments.append(rex) else: - final.append(segment) - return final + split_compiled_segments.append(segment) + + return split_compiled_segments def new(obj: MutableMapping, path: Path, value, separator="/", creator: Creator = None) -> MutableMapping: @@ -87,7 +98,7 @@ def new(obj: MutableMapping, path: Path, value, separator="/", creator: Creator return segments.set(obj, split_segments, value) -def delete(obj: MutableMapping, glob: Glob, separator="/", afilter: Filter = None) -> int: +def delete(obj: MutableMapping, glob: GlobExtend, separator="/", afilter: Filter = None) -> int: """ Given a obj, delete all elements that match the glob. @@ -146,7 +157,7 @@ def f(obj, pair, counter): return deleted -def set(obj: MutableMapping, glob: Glob, value, separator="/", afilter: Filter = None) -> int: +def set(obj: MutableMapping, glob: GlobExtend, value, separator="/", afilter: Filter = None) -> int: """ Given a path glob, set all existing elements in the document to the given value. Returns the number of elements changed. @@ -173,7 +184,7 @@ def f(obj, pair, counter): def get( obj: MutableMapping, - glob: Glob, + glob: GlobExtend, separator="/", default: Any = _DEFAULT_SENTINEL ) -> Union[MutableMapping, object, Callable]: @@ -212,7 +223,7 @@ def f(_, pair, results): return results[0] -def values(obj: MutableMapping, glob: Glob, separator="/", afilter: Filter = None, dirs=True): +def values(obj: MutableMapping, glob: GlobExtend, separator="/", afilter: Filter = None, dirs=True): """ Given an object and a path glob, return an array of all values which match the glob. The arguments to this function are identical to those of search(). @@ -222,7 +233,7 @@ def values(obj: MutableMapping, glob: Glob, separator="/", afilter: Filter = Non return [v for p, v in search(obj, glob, yielded, separator, afilter, dirs)] -def search(obj: MutableMapping, glob: Glob, yielded=False, separator="/", afilter: Filter = None, dirs=True): +def search(obj: MutableMapping, glob: GlobExtend, yielded=False, separator="/", afilter: Filter = None, dirs=True): """ Given a path glob, return a dictionary containing all keys that matched the given glob. diff --git a/dpath/exceptions.py b/dpath/exceptions.py index 3b1a7da..22f7a13 100644 --- a/dpath/exceptions.py +++ b/dpath/exceptions.py @@ -3,6 +3,11 @@ class InvalidGlob(Exception): pass +class InvalidRegex(Exception): + """Erroneous re regular expression in path segment """ + pass + + class PathNotFound(Exception): """One or more elements of the requested path did not exist in the object""" pass diff --git a/dpath/options.py b/dpath/options.py index 260d922..1ec30bd 100644 --- a/dpath/options.py +++ b/dpath/options.py @@ -1,7 +1,8 @@ ALLOW_EMPTY_STRING_KEYS = False -# Extension to interpret path segments "{rrr}" as re.regexp "rrr" enabled by default. +# Extension to interpret path segments "{rrr}" as re.regexp "rrr" disabled by default. # Disable to preserve backwards compatibility in the case where a user has a # path "a/b/{cd}" where the brackets are intentional and do not denote a request # to re.compile cd -DPATH_ACCEPT_RE_REGEXP = True +# Enable to allow segment matching with Python re regular expressions. +DPATH_ACCEPT_RE_REGEXP_IN_STRING = False diff --git a/dpath/segments.py b/dpath/segments.py index 46deeee..1643f4d 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -1,13 +1,12 @@ from copy import deepcopy from fnmatch import fnmatchcase from typing import Sequence, Tuple, Iterator, Any, Union, Optional, MutableMapping, MutableSequence +from re import Pattern from dpath import options from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound from dpath.types import PathSegment, Creator, Hints, Glob, Path, SymmetricInt -from re import Pattern - def make_walkable(node) -> Iterator[Tuple[PathSegment, Any]]: """ @@ -38,7 +37,7 @@ def leaf(thing): """ Return True if thing is a leaf, otherwise False. """ - leaves = (bytes, str, int, float, bool, type(None)) + leaves = (bytes, str, int, float, bool, type(None), Pattern) return isinstance(thing, leaves) diff --git a/dpath/types.py b/dpath/types.py index c4a4a56..fc9c68e 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -1,5 +1,6 @@ from enum import IntFlag, auto from typing import Union, Any, Callable, Sequence, Tuple, List, Optional, MutableMapping +from re import Pattern class SymmetricInt(int): @@ -57,6 +58,19 @@ class MergeType(IntFlag): Glob = Union[str, Sequence[str]] """Type alias for glob parameters.""" + +GlobExtend = Union[str, Pattern, Sequence[Union[str, Pattern]]] +""" +Type alias for globs extended by the possibility to have re.Pattern. +Note that this is a subset of Path, but it seems that changing +typing from Glob to GlobExtend is sufficient. + +Main reason for allowing Pattern is to treat similarly the path '{regex}' and +re.compile("regex"). + +This is tested in tests.test_various_exts. +""" + Path = Union[str, Sequence[PathSegment]] """Type alias for path parameters.""" diff --git a/tests/test_various_exts.py b/tests/test_various_exts.py index ac36779..979f995 100644 --- a/tests/test_various_exts.py +++ b/tests/test_various_exts.py @@ -14,13 +14,14 @@ import unittest import dpath as DP +from dpath.exceptions import InvalidRegex # check that how the options have been set -print(f"At entry in test_path_ext DPATH_ACCEPT_RE_REGEXP = {DP.options.DPATH_ACCEPT_RE_REGEXP}", file=sys.stderr) +print(f"At entry in test_path_ext DPATH_ACCEPT_RE_REGEXP_IN_STRING = {DP.options.DPATH_ACCEPT_RE_REGEXP_IN_STRING}", file=sys.stderr) -if not DP.options.DPATH_ACCEPT_RE_REGEXP: - print("This test doesn't make sense with DPATH_ACCEPT_RE_REGEXP = False", file=sys.stderr) - DP.options.DPATH_ACCEPT_RE_REGEXP = True # enable re.regexp support in path expr. +if not DP.options.DPATH_ACCEPT_RE_REGEXP_IN_STRING: + print("switching to DPATH_ACCEPT_RE_REGEXP_IN_STRING = True", file=sys.stderr) + DP.options.DPATH_ACCEPT_RE_REGEXP_IN_STRING = True # enable re.regexp support in path expr. # one class per function to be tested @@ -39,15 +40,15 @@ def build(self): }, } - self.specs1 = (([re.compile(".*")], "a001", True), - ([re.compile("[a-z]+$")], "a001", False), - (["*", re.compile(".*")], "a001", False), - (["*", "*", re.compile(".*")], "a001", False), - (["*", re.compile("[a-z]+\\d+$")], "a001", False), - (["*", re.compile("[a-z]+[.][a-z]+$")], "a001", False), - (["**", re.compile(".*")], "a001", True), - (["**", re.compile("[a-z]+\\d+$")], "a001", True), - (["**", re.compile("[a-z]+[.][a-z]+$")], "a001", False) + self.specs1 = (([re.compile(".*")], "a001", True, set()), + ([re.compile("[a-z]+$")], "a001", False, set()), + (["*", re.compile(".*")], "a001", False, set()), + (["*", "*", re.compile(".*")], "a001", False, set()), + (["*", re.compile("[a-z]+\\d+$")], "a001", False, set()), + (["*", re.compile("[a-z]+[.][a-z]+$")], "a001", False, set()), + (["**", re.compile(".*")], "a001", True, set((0,1,2))), + (["**", re.compile("[a-z]+\\d+$")], "a001", True, set()), + (["**", re.compile("[a-z]+[.][a-z]+$")], "a001", False, set((0,1))) ) self.specs1Pairs = (([re.compile(".*")], ("a001",)), @@ -123,8 +124,7 @@ def build(self): ("Name", "Id", "Created", "Scope", "Driver", "Internal", "Attachable", "Ingress", "Containers", "Options", "Labels", "IPAM/Driver", "IPAM/Options", "IPAM/Config", "IPAM/Config/Subnet", "IPAM/Config/Gateway", - "ConfigFrom/Network", "Containers/199c590e8f13477/Name", - "Containers/199c590e8f13477/MacAddress")), + "ConfigFrom/Network", "Containers/199c590e8f13477/Name")), (["**", re.compile("[A-Z][A-Za-z\\d]*Address$")], ("Containers/199c590e8f13477/MacAddress", "Containers/199c590e8f13477/IPv4Address", "Containers/199c590e8f13477/IPv6Address")), @@ -139,8 +139,7 @@ def build(self): ("Name", "Id", "Created", "Scope", "Driver", "Internal", "Attachable", "Ingress", "Containers", "Options", "Labels", "IPAM/Driver", "IPAM/Options", "IPAM/Config", "IPAM/Config/Subnet", "IPAM/Config/Gateway", - "ConfigFrom/Network", "Containers/199c590e8f13477/Name", - "Containers/199c590e8f13477/MacAddress")), + "ConfigFrom/Network", "Containers/199c590e8f13477/Name")), (["**", re.compile(r"[A-Z][A-Za-z\d]*Address$")], ("Containers/199c590e8f13477/MacAddress", "Containers/199c590e8f13477/IPv4Address", "Containers/199c590e8f13477/IPv6Address")), @@ -148,12 +147,10 @@ def build(self): (["**", re.compile(r"\d+[.]\d+")], None) ) - self.specs3Pairs = (("**/{[^A-Za-z]{2}$}", ("Id",)), - ("*/{[A-Z][A-Za-z\\d]*$}", ("Name", "Id", "Created", "Scope", "Driver", "Internal", - "Attachable", "Ingress", "Containers", "Options", "Labels", - "IPAM/Driver", "IPAM/Options", "IPAM/Config", "IPAM/Config/Subnet", - "IPAM/Config/Gateway", "ConfigFrom/Network", "Containers/199c590e8f13477/Name", - "Containers/199c590e8f13477/MacAddress")), + self.specs3Pairs = (("**/{^[A-Za-z]{2}$}", ("Id",)), + ("{^[A-Za-z]{2}$}", ("Id",)), + (re.compile("^[A-Za-z]{2}$"), ("Id",)), + ("*/{[A-Z][A-Za-z\\d]*$}", ("IPAM/Driver", "IPAM/Options", "IPAM/Config", "ConfigFrom/Network")), ("**/{[A-Z][A-Za-z\\d]*\\d$}", ("EnableIPv6",)), ("**/{[A-Z][A-Za-z\\d]*Address$}", ("Containers/199c590e8f13477/MacAddress", "Containers/199c590e8f13477/IPv4Address", @@ -172,26 +169,36 @@ def build(self): class TestSearch(unittest.TestCase): def test1(self): - print("Entered test1", file=sys.__stderr__) + print("Entered test1", file=sys.stderr) dicts = SampleDicts().build() dict1 = dicts.d1 specs = dicts.specs1Pairs for (spec, expect) in specs: print(f"Spec={spec}", file=sys.stderr) + found = set() for (path, value) in DP.search(dict1, spec, yielded=True): print(f"\tpath={path}\tv={value}\n\texpected={expect}", file=sys.stderr) if path is None: assert expect is None else: + found.add(path) assert (path in expect) - print("\n", file=sys.stderr) + if expect is not None: + diff = found ^ set(expect) + if len(diff) != 0: + print(f"Error\t{found=}\n\t{expect=}", file=sys.stderr) + print(f"Symmetric Difference : {diff}", file=sys.stderr) + assert False + else: + assert len(found) == 0 + def test2(self): - print("Entered test2", file=sys.__stderr__) + print("Entered test2", file=sys.stderr) + print(f"Test for filtering for int values", file=sys.stderr) def afilter(x): - # print(f"In afilter x = {x}({type(x)})", file=sys.stderr) if isinstance(x, int): return True return False @@ -199,45 +206,91 @@ def afilter(x): dicts = SampleDicts().build() dict1 = dicts.d1 specs = dicts.specs1 - for spec in (s[0] for s in specs): - print(f"Spec={spec}", file=sys.stderr) + for spec, expected in ( (s[0],s[3]) for s in specs): + print(f"Spec={spec}, Key={expected}", file=sys.stderr) + result = set() for ret in DP.search(dict1, spec, yielded=True, afilter=afilter): print(f"\tret={ret}", file=sys.stderr) - assert (isinstance(ret[1], int)) + result.add(ret[1]) + assert isinstance(ret[1], int) + assert result == expected def test3(self): - print("Entered test3", file=sys.__stderr__) + print("Entered test3", file=sys.stderr) dicts = SampleDicts().build() dict1 = dicts.d2 specs = dicts.specs2Pairs for (spec, expect) in specs: print(f"Spec={spec}", file=sys.stderr) + found = set() for (path, value) in DP.search(dict1, spec, yielded=True): print(f"\tpath={path}\tv={value}", file=sys.stderr) if path is None: assert expect is None else: + found.add(path) assert (path in expect) + if expect is not None: + diff = found ^ set(expect) + if len(diff) != 0: + print(f"Error\t{found=}\n\t{expect=}", file=sys.stderr) + print(f"Symmetric Difference : {diff}", file=sys.stderr) + assert False + else: + assert len(found) == 0 def test4(self): - print("Entered test4", file=sys.__stderr__) + print("Entered test4", file=sys.stderr) dicts = SampleDicts().build() dict1 = dicts.d2 specs = dicts.specs3Pairs for (spec, expect) in specs: print(f"Spec={spec}", file=sys.stderr) + found = set() for (path, value) in DP.search(dict1, spec, yielded=True): print(f"\tpath={path}\tv={value}", file=sys.stderr) if path is None: assert expect is None else: + found.add(path) assert (path in expect) + if expect is not None: + diff = found ^ set(expect) + if len(diff) != 0: + print(f"Error\t{found=}\n\t{expect=}", file=sys.stderr) + print(f"Symmetric Difference : {diff}", file=sys.stderr) + assert False + else: + assert len(found) == 0 + + def test5(self): + print("Entered test5 -- re.error::", file=sys.stderr) + dicts = SampleDicts().build() + dict1 = dicts.d2 + specs = (("/**/{zz)bad}", "ERROR"), + ("{zz)bad}/yyy", "ERROR"), + ("**/{zz)bad}/yyy", "ERROR"), + ("**/{zz)bad}/yyy/.*", "ERROR"), + (123, "OTHERERROR")) + + for (spec, expect) in specs: + print(f"Spec={spec}", file=sys.stderr) + try: + for (path, value) in DP.search(dict1, spec, yielded=True): + print(f"\tpath={path}\tv={value}", file=sys.stderr) + assert expect not in ("ERROR", "OTHERERROR") + except InvalidRegex as errExpected: + print(f"Expected error:{errExpected}", file=sys.stderr) + assert expect == "ERROR" + except Exception as errExpected: + print(f"Expected error:{errExpected}", file=sys.stderr) + assert expect == "OTHERERROR" class TestGet(unittest.TestCase): def test1(self): - print("Entered test1", file=sys.__stderr__) + print("Entered test1", file=sys.stderr) dicts = SampleDicts().build() dict1 = dicts.d1 @@ -248,7 +301,7 @@ def test1(self): ret = DP.get(dict1, spec, default=("*NONE*",)) print(f"\tret={ret}", file=sys.stderr) assert (ret == expect) - except Exception as err: + except ValueError as err: print("\t get fails:", err, type(err), file=sys.stderr) assert (expect == "*FAIL*") @@ -284,7 +337,7 @@ def test1(self): class TestView(unittest.TestCase): def test1(self): - print("Entered test1", file=sys.__stderr__) + print("Entered test1", file=sys.stderr) dicts = SampleDicts().build() dict1 = dicts.d1 @@ -297,7 +350,7 @@ def test1(self): class TestMatch(unittest.TestCase): def test1(self): - print("Entered test1", file=sys.__stderr__) + print("Entered test1", file=sys.stderr) dicts = SampleDicts().build() dict1 = dicts.d1 From 303eb09af79d037b562a7b2faa841191f29c0584 Mon Sep 17 00:00:00 2001 From: Alain Lich Date: Sat, 15 Apr 2023 22:25:41 +0200 Subject: [PATCH 07/16] Test correction, testability enhancements - made option DPATH_ACCEPT_RE_REGEXP_IN_STRING configurable from sys. environ - separated in test_various_exts tests which require this to be True, other tests may always be run - added tox-set-rex.ini which sets (whereas the default is unset) - python3Test.yml runs both tox configurations, reduced number of Python & Pypy configs --- .github/workflows/python3Test.yml | 43 +++-- dpath/options.py | 22 +++ tests/test_various_exts.py | 278 +++++++++++++++--------------- tox-set-rex.ini | 31 ++++ tox.ini | 4 +- 5 files changed, 212 insertions(+), 166 deletions(-) create mode 100644 tox-set-rex.ini diff --git a/.github/workflows/python3Test.yml b/.github/workflows/python3Test.yml index 404a445..1a5a67a 100644 --- a/.github/workflows/python3Test.yml +++ b/.github/workflows/python3Test.yml @@ -4,9 +4,7 @@ name: Test python package dpath-python with regexp extension # # For running under Github's Actions # - # Script performs basic test of the Python-3 version of the package - # including added functionality (regexp in search paths). - # ------------------------------------------------------------ + # ------------------------------------------------------------ # ***************************** # ADDED FOR TESTING PRIOR TO PR @@ -26,8 +24,8 @@ jobs: strategy: matrix: - # Match versions specified in tox.ini - python-version: ['3.8', '3.9', '3.10','3.11', 'pypy3.7', 'pypy3.9'] + # Match versions specified in tox.ini and tox-set-rex.ini + python-version: ['3.8', '3.11', 'pypy3.7', 'pypy3.9'] steps: - name: Checkout code @@ -53,15 +51,6 @@ jobs: echo LD_LIBRARY_PATH: ${LD_LIBRARY_PATH} echo PYTHONPATH: \'${PYTHONPATH}\' - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - # ** here (it is expected that) ** - # pythonLocation: /opt/hostedtoolcache/Python/3.8.8/x64 - # LD_LIBRARY_PATH: /opt/hostedtoolcache/Python/3.8.8/x64/lib - # Working dir /home/runner/work/dpath-python/dpath-python - # HOME: /home/runner - # LANG: C.UTF-8 - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - - name: Install dependencies shell: bash if: always() @@ -76,20 +65,28 @@ jobs: pip install -r requirements.txt; fi python setup.py install - echo which nose :$(which nose) - echo which nose2: $(which nose2) - echo which nose2-3.6: $(which nose2-3.6) - echo which nose2-3.8: $(which nose2-3.8) + pip install tox + echo "Installed tox" - - name: Tox testing + - name: Tox test with default DPATH_ACCEPT_RE_REGEXP_IN_STRING = FALSE shell: bash if: always() # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - # move to tox testing, otherwise will have to parametrize - # nose in more details; here tox.ini will apply + # tox testing, here tox.ini is used + # DPATH_ACCEPT_RE_REGEXP_IN_STRING = FALSE (default) # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ run: | - pip install tox - echo "Installed tox" tox echo "Ran tox" + + - name: Tox test with DPATH_ACCEPT_RE_REGEXP_IN_STRING = TRUE + shell: bash + if: always() + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # tox testing, here tox-set-rex.ini is used + # DPATH_ACCEPT_RE_REGEXP_IN_STRING = TRUE + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + run: | + tox -c tox-set-rex.ini + echo "Ran tox -c tox-set-rex.ini" + \ No newline at end of file diff --git a/dpath/options.py b/dpath/options.py index 1ec30bd..d8721f1 100644 --- a/dpath/options.py +++ b/dpath/options.py @@ -1,3 +1,5 @@ +from os import environ + ALLOW_EMPTY_STRING_KEYS = False # Extension to interpret path segments "{rrr}" as re.regexp "rrr" disabled by default. @@ -5,4 +7,24 @@ # path "a/b/{cd}" where the brackets are intentional and do not denote a request # to re.compile cd # Enable to allow segment matching with Python re regular expressions. + DPATH_ACCEPT_RE_REGEXP_IN_STRING = False + +# ---------------------------------------------------------------------------------------------- +# undocumented feature for testability: facilitate running the entire package (or test suite) +# with option enabled/disabled by setting "DPATH_ACCEPT_RE_REGEXP_IN_STRING" in process environment. +# Value => Effect +# TRUE : set to True +# TRUE_PRINT: set to True and output confirmation to stderr +# FALSE: set to False +# FALSE_PRINT: set to False and output confirmation to stderr +if "DPATH_ACCEPT_RE_REGEXP_IN_STRING" in environ: + setTrue = environ["DPATH_ACCEPT_RE_REGEXP_IN_STRING"] + if setTrue in ("TRUE", "TRUE_PRINT"): + DPATH_ACCEPT_RE_REGEXP_IN_STRING = True + else: + DPATH_ACCEPT_RE_REGEXP_IN_STRING = False + + if setTrue[-5:] == "PRINT": + from sys import stderr + print(f"DPATH_ACCEPT_RE_REGEXP_IN_STRING={DPATH_ACCEPT_RE_REGEXP_IN_STRING}", file=stderr) diff --git a/tests/test_various_exts.py b/tests/test_various_exts.py index 979f995..a5ff8b1 100644 --- a/tests/test_various_exts.py +++ b/tests/test_various_exts.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # -*- mode: Python -*- # -# (C) Alain Lichnewsky, 2022 +# (C) Alain Lichnewsky, 2022, 2023 # # Test support of extended specs with re.regex in many functionalities that use path # specifications. @@ -19,12 +19,7 @@ # check that how the options have been set print(f"At entry in test_path_ext DPATH_ACCEPT_RE_REGEXP_IN_STRING = {DP.options.DPATH_ACCEPT_RE_REGEXP_IN_STRING}", file=sys.stderr) -if not DP.options.DPATH_ACCEPT_RE_REGEXP_IN_STRING: - print("switching to DPATH_ACCEPT_RE_REGEXP_IN_STRING = True", file=sys.stderr) - DP.options.DPATH_ACCEPT_RE_REGEXP_IN_STRING = True # enable re.regexp support in path expr. - -# one class per function to be tested class SampleDicts(): def build(self): @@ -46,10 +41,9 @@ def build(self): (["*", "*", re.compile(".*")], "a001", False, set()), (["*", re.compile("[a-z]+\\d+$")], "a001", False, set()), (["*", re.compile("[a-z]+[.][a-z]+$")], "a001", False, set()), - (["**", re.compile(".*")], "a001", True, set((0,1,2))), + (["**", re.compile(".*")], "a001", True, set((0, 1, 2))), (["**", re.compile("[a-z]+\\d+$")], "a001", True, set()), - (["**", re.compile("[a-z]+[.][a-z]+$")], "a001", False, set((0,1))) - ) + (["**", re.compile("[a-z]+[.][a-z]+$")], "a001", False, set((0, 1)))) self.specs1Pairs = (([re.compile(".*")], ("a001",)), ([re.compile("[a-z]+$")], None), @@ -58,11 +52,11 @@ def build(self): (["*", re.compile("[a-z]+\\d+$")], ("a001/b2",)), (["*", re.compile("[a-z]+[.][a-z]+$")], None), (["**", re.compile(".*")], - ("a001", "a001/b2", "a001/b2/c1.2", "a001/b2/c1.2/d.dd", - "a001/b2/c1.2/e.ee", "a001/b2/c1.2/f.f0")), + ("a001", "a001/b2", "a001/b2/c1.2", "a001/b2/c1.2/d.dd", + "a001/b2/c1.2/e.ee", "a001/b2/c1.2/f.f0")), (["**", re.compile("[a-z]+\\d+$")], ("a001", "a001/b2")), (["**", re.compile("[a-z]+[.][a-z]+$")], - ("a001/b2/c1.2/d.dd", "a001/b2/c1.2/e.ee")) + ("a001/b2/c1.2/d.dd", "a001/b2/c1.2/e.ee")) ) self.specs1GetPairs = (([re.compile(".*")], {'b2': {'c1.2': {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}}}), @@ -73,50 +67,44 @@ def build(self): (["*", re.compile("[a-z]+[.][a-z]+$")], ('*NONE*',)), (["**", re.compile(".*")], "*FAIL*"), (["**", re.compile("[a-z]+\\d+$")], "*FAIL*"), - (["**", re.compile("[a-z]+[.][a-z]+$")], "*FAIL*"), - ) - - self.d2 = {"Name": "bridge", - "Id": "333d22b3724", - "Created": "2022-12-08T09:02:33.360812052+01:00", - "Scope": "local", - "Driver": "bridge", - "EnableIPv6": False, - "IPAM": { - "Driver": "default", - "Options": None, - "Config": - { - "Subnet": "172.17.0.0/16", - "Gateway": "172.17.0.1" - } - }, - "Internal": False, - "Attachable": False, - "Ingress": False, - "ConfigFrom": { - "Network": "" - }, - "ConfigOnly": False, - "Containers": { - "199c590e8f13477": { - "Name": "al_dpath", - "EndpointID": "3042bbe16160a63b7", - "MacAddress": "02:34:0a:11:10:22", - "IPv4Address": "172.17.0.2/16", - "IPv6Address": "" - } - }, - "Options": { - "com.docker.network.bridge.default_bridge": "true", - "com.docker.network.bridge.enable_icc": "true", - "com.docker.network.bridge.enable_ip_masquerade": "true", - "com.docker.network.bridge.host_binding_ipv4": "0.0.0.0", - "com.docker.network.bridge.name": "docker0", - "com.docker.network.driver.mtu": "1500" - }, - "Labels": {} - } + (["**", re.compile("[a-z]+[.][a-z]+$")], "*FAIL*"),) + + self.d2 = { + "Name": "bridge", + "Id": "333d22b3724", + "Created": "2022-12-08T09:02:33.360812052+01:00", + "Scope": "local", + "Driver": "bridge", + "EnableIPv6": False, + "IPAM": { + "Driver": "default", + "Options": None, + "Config": + { + "Subnet": "172.17.0.0/16", + "Gateway": "172.17.0.1" + }}, + "Internal": False, + "Attachable": False, + "Ingress": False, + "ConfigFrom": { + "Network": ""}, + "ConfigOnly": False, + "Containers": { + "199c590e8f13477": { + "Name": "al_dpath", + "EndpointID": "3042bbe16160a63b7", + "MacAddress": "02:34:0a:11:10:22", + "IPv4Address": "172.17.0.2/16", + "IPv6Address": ""}}, + "Options": { + "com.docker.network.bridge.default_bridge": "true", + "com.docker.network.bridge.enable_icc": "true", + "com.docker.network.bridge.enable_ip_masquerade": "true", + "com.docker.network.bridge.host_binding_ipv4": "0.0.0.0", + "com.docker.network.bridge.name": "docker0", + "com.docker.network.driver.mtu": "1500"}, + "Labels": {}} self.specs2Pairs = ((["*", re.compile("[A-Z][a-z\\d]*$")], ("IPAM/Driver", "IPAM/Options", "IPAM/Config", "ConfigFrom/Network")), @@ -166,8 +154,11 @@ def build(self): return self -class TestSearch(unittest.TestCase): +# one class per function to be tested, postpone tests that need +# DP.options.DPATH_ACCEPT_RE_REGEXP_IN_STRING == True + +class TestSearchAlways(unittest.TestCase): def test1(self): print("Entered test1", file=sys.stderr) dicts = SampleDicts().build() @@ -187,16 +178,15 @@ def test1(self): if expect is not None: diff = found ^ set(expect) if len(diff) != 0: - print(f"Error\t{found=}\n\t{expect=}", file=sys.stderr) + print(f"Error\tFound:{found}\n\tExpected:{expect}", file=sys.stderr) print(f"Symmetric Difference : {diff}", file=sys.stderr) assert False else: - assert len(found) == 0 - + assert len(found) == 0 def test2(self): print("Entered test2", file=sys.stderr) - print(f"Test for filtering for int values", file=sys.stderr) + print("Test for filtering for int values", file=sys.stderr) def afilter(x): if isinstance(x, int): @@ -206,7 +196,7 @@ def afilter(x): dicts = SampleDicts().build() dict1 = dicts.d1 specs = dicts.specs1 - for spec, expected in ( (s[0],s[3]) for s in specs): + for spec, expected in ((s[0], s[3]) for s in specs): print(f"Spec={spec}, Key={expected}", file=sys.stderr) result = set() for ret in DP.search(dict1, spec, yielded=True, afilter=afilter): @@ -233,59 +223,12 @@ def test3(self): if expect is not None: diff = found ^ set(expect) if len(diff) != 0: - print(f"Error\t{found=}\n\t{expect=}", file=sys.stderr) - print(f"Symmetric Difference : {diff}", file=sys.stderr) - assert False - else: - assert len(found) == 0 - - def test4(self): - print("Entered test4", file=sys.stderr) - dicts = SampleDicts().build() - dict1 = dicts.d2 - specs = dicts.specs3Pairs - for (spec, expect) in specs: - print(f"Spec={spec}", file=sys.stderr) - found = set() - for (path, value) in DP.search(dict1, spec, yielded=True): - print(f"\tpath={path}\tv={value}", file=sys.stderr) - if path is None: - assert expect is None - else: - found.add(path) - assert (path in expect) - if expect is not None: - diff = found ^ set(expect) - if len(diff) != 0: - print(f"Error\t{found=}\n\t{expect=}", file=sys.stderr) - print(f"Symmetric Difference : {diff}", file=sys.stderr) + print(f"Error\tFound:{found}\n\tExpected:{expect}", file=sys.stderr) + print(f"Symmetric Difference: {diff}", file=sys.stderr) assert False else: assert len(found) == 0 - def test5(self): - print("Entered test5 -- re.error::", file=sys.stderr) - dicts = SampleDicts().build() - dict1 = dicts.d2 - specs = (("/**/{zz)bad}", "ERROR"), - ("{zz)bad}/yyy", "ERROR"), - ("**/{zz)bad}/yyy", "ERROR"), - ("**/{zz)bad}/yyy/.*", "ERROR"), - (123, "OTHERERROR")) - - for (spec, expect) in specs: - print(f"Spec={spec}", file=sys.stderr) - try: - for (path, value) in DP.search(dict1, spec, yielded=True): - print(f"\tpath={path}\tv={value}", file=sys.stderr) - assert expect not in ("ERROR", "OTHERERROR") - except InvalidRegex as errExpected: - print(f"Expected error:{errExpected}", file=sys.stderr) - assert expect == "ERROR" - except Exception as errExpected: - print(f"Expected error:{errExpected}", file=sys.stderr) - assert expect == "OTHERERROR" - class TestGet(unittest.TestCase): @@ -306,35 +249,6 @@ def test1(self): assert (expect == "*FAIL*") -class TestDelete(unittest.TestCase): - def test1(self): - print("This is test1", file=sys.stderr) - dict1 = { - "a": { - "b": 0, - "12": 0, - }, - "a0": { - "b": 0, - }, - } - - specs = (re.compile("[a-z]+$"), re.compile("[a-z]+\\d+$"), - "{[a-z]+\\d+$}") - i = 0 - for spec in specs: - dict = copy(dict1) - print(f"spec={spec}") - print(f"Before deletion dict={dict}", file=sys.stderr) - DP.delete(dict, [spec]) - print(f"After deletion dict={dict}", file=sys.stderr) - if i == 0: - assert (dict == {"a0": {"b": 0, }, }) - else: - assert (dict == {"a": {"b": 0, "12": 0, }}) - i += 1 - - class TestView(unittest.TestCase): def test1(self): print("Entered test1", file=sys.stderr) @@ -359,3 +273,87 @@ def test1(self): ret = DP.segments.match(dict1, spec[0]) print(f"\tmatch spec:{spec} returns:{ret}", file=sys.stderr) assert ret == spec[2] + + +# Now tests that require DP.options.DPATH_ACCEPT_RE_REGEXP_IN_STRING to be set +if not DP.options.DPATH_ACCEPT_RE_REGEXP_IN_STRING: + print("Skipping because DPATH_ACCEPT_RE_REGEXP_IN_STRING != True", file=sys.stderr) + +else: + + class TestSearch(unittest.TestCase): + + def test4(self): + print("Entered test4", file=sys.stderr) + dicts = SampleDicts().build() + dict1 = dicts.d2 + specs = dicts.specs3Pairs + for (spec, expect) in specs: + print(f"Spec={spec}", file=sys.stderr) + found = set() + for (path, value) in DP.search(dict1, spec, yielded=True): + print(f"\tpath={path}\tv={value}", file=sys.stderr) + if path is None: + assert expect is None + else: + found.add(path) + assert (path in expect) + if expect is not None: + diff = found ^ set(expect) + if len(diff) != 0: + print(f"Error\tFound:{found}\n\tExpected:{expect}", file=sys.stderr) + print(f"Symmetric Difference : {diff}", file=sys.stderr) + assert False + else: + assert len(found) == 0 + + def test5(self): + print("Entered test5 -- re.error::", file=sys.stderr) + dicts = SampleDicts().build() + dict1 = dicts.d2 + specs = (("/**/{zz)bad}", "ERROR"), + ("{zz)bad}/yyy", "ERROR"), + ("**/{zz)bad}/yyy", "ERROR"), + ("**/{zz)bad}/yyy/.*", "ERROR"), + (123, "OTHERERROR")) + + for (spec, expect) in specs: + print(f"Spec={spec}", file=sys.stderr) + try: + for (path, value) in DP.search(dict1, spec, yielded=True): + print(f"\tpath={path}\tv={value}", file=sys.stderr) + assert expect not in ("ERROR", "OTHERERROR") + except InvalidRegex as errExpected: + print(f"Expected error:{errExpected}", file=sys.stderr) + assert expect == "ERROR" + except Exception as errExpected: + print(f"Expected error:{errExpected}", file=sys.stderr) + assert expect == "OTHERERROR" + + class TestDelete(unittest.TestCase): + def test1(self): + print("This is test1", file=sys.stderr) + dict1 = { + "a": { + "b": 0, + "12": 0, + }, + "a0": { + "b": 0, + }, + } + + specs = (re.compile("[a-z]+$"), re.compile("[a-z]+\\d+$"), + "{[a-z]+\\d+$}") + i = 0 + for spec in specs: + dict = copy(dict1) + print(f"spec={spec}") + print(f"Before deletion dict={dict}", file=sys.stderr) + DP.delete(dict, [spec]) + print(f"After deletion dict={dict}", file=sys.stderr) + if i == 0: + assert (dict == {"a0": {"b": 0, }, }) + else: + assert (dict == {"a": {"b": 0, "12": 0, }}) + i += 1 diff --git a/tox-set-rex.ini b/tox-set-rex.ini new file mode 100644 index 0000000..4eca5fe --- /dev/null +++ b/tox-set-rex.ini @@ -0,0 +1,31 @@ +# ******** THIS VERSION ENABLES DPATH_ACCEPT_RE_REGEXP_IN_STRING +# ******** + +# Tox (http://tox.testrun.org/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + + +[flake8] +ignore = E501,E722,W503 + +[tox] +envlist = pypy37, pypy39, py38, py311 + +[gh-actions] +python = + pypy-3.7: pypy37 + pypy-3.9: pypy39 + 3.8: py38 + 3.11: py311 + +[testenv] +deps = + hypothesis + mock + nose2 +commands = nose2 {posargs} +setenv = + DPATH_ACCEPT_RE_REGEXP_IN_STRING = TRUE + diff --git a/tox.ini b/tox.ini index abe73a8..eaeca41 100644 --- a/tox.ini +++ b/tox.ini @@ -7,15 +7,13 @@ ignore = E501,E722,W503 [tox] -envlist = pypy37, pypy39, py38, py39, py310, py311 +envlist = pypy37, pypy39, py38, py311 [gh-actions] python = pypy-3.7: pypy37 pypy-3.9: pypy39 3.8: py38 - 3.9: py39 - 3.10: py310 3.11: py311 [testenv] From 3363da2e7dcd3864851cbd18c69904ba9f0b7eec Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 16 Apr 2023 01:50:18 +0300 Subject: [PATCH 08/16] Wording and minor changes --- dpath/__init__.py | 2 +- dpath/exceptions.py | 8 ++++---- dpath/options.py | 12 ++++++------ dpath/segments.py | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/dpath/__init__.py b/dpath/__init__.py index e15f03a..ff40ec8 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -20,9 +20,9 @@ "Creator", ] +import re from collections.abc import MutableMapping, MutableSequence from typing import Union, List, Any, Callable, Optional -import re from dpath import segments, options from dpath.exceptions import InvalidKeyName, PathNotFound, InvalidRegex diff --git a/dpath/exceptions.py b/dpath/exceptions.py index 22f7a13..cc85313 100644 --- a/dpath/exceptions.py +++ b/dpath/exceptions.py @@ -4,20 +4,20 @@ class InvalidGlob(Exception): class InvalidRegex(Exception): - """Erroneous re regular expression in path segment """ + """Invalid regular expression in path segment.""" pass class PathNotFound(Exception): - """One or more elements of the requested path did not exist in the object""" + """One or more elements of the requested path did not exist in the object.""" pass class InvalidKeyName(Exception): - """This key contains the separator character or another invalid character""" + """This key contains the separator character or another invalid character.""" pass class FilteredValue(Exception): - """Unable to return a value, since the filter rejected it""" + """Unable to return a value, since the filter rejected it.""" pass diff --git a/dpath/options.py b/dpath/options.py index d8721f1..fe2a34c 100644 --- a/dpath/options.py +++ b/dpath/options.py @@ -2,13 +2,13 @@ ALLOW_EMPTY_STRING_KEYS = False -# Extension to interpret path segments "{rrr}" as re.regexp "rrr" disabled by default. -# Disable to preserve backwards compatibility in the case where a user has a -# path "a/b/{cd}" where the brackets are intentional and do not denote a request -# to re.compile cd -# Enable to allow segment matching with Python re regular expressions. - DPATH_ACCEPT_RE_REGEXP_IN_STRING = False +"""Enables regular expression support. + +Enabling this feature will allow usage of regular expressions as part of paths. +Regular expressions must be wrapped in curly brackets. For example: "a/b/{[cd]}". +Expressions will be compiled using the standard library re.compile function. +""" # ---------------------------------------------------------------------------------------------- # undocumented feature for testability: facilitate running the entire package (or test suite) diff --git a/dpath/segments.py b/dpath/segments.py index 1643f4d..6dd0f94 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -1,7 +1,7 @@ from copy import deepcopy from fnmatch import fnmatchcase -from typing import Sequence, Tuple, Iterator, Any, Union, Optional, MutableMapping, MutableSequence from re import Pattern +from typing import Sequence, Tuple, Iterator, Any, Union, Optional, MutableMapping, MutableSequence from dpath import options from dpath.exceptions import InvalidGlob, InvalidKeyName, PathNotFound From 3b4eac66131efc482f46e9c7181906b9e20bf6d6 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 16 Apr 2023 01:51:45 +0300 Subject: [PATCH 09/16] Integration of re.Pattern into existing types --- dpath/__init__.py | 15 +++++++-------- dpath/types.py | 18 +++--------------- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/dpath/__init__.py b/dpath/__init__.py index ff40ec8..6361695 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -26,13 +26,12 @@ from dpath import segments, options from dpath.exceptions import InvalidKeyName, PathNotFound, InvalidRegex -from dpath.types import MergeType, PathSegment, Creator, Filter, Glob, GlobExtend, Path, Hints - +from dpath.types import MergeType, PathSegment, Creator, Filter, Glob, Path, Hints _DEFAULT_SENTINEL = object() -def _split_path(path: GlobExtend, separator: Optional[str] = "/") -> Union[List[PathSegment], PathSegment]: +def _split_path(path: Glob, separator: Optional[str] = "/") -> Union[List[PathSegment], PathSegment]: """ Given a path and separator, return a tuple of segments; string segments that represent (i.e. "{.*}") regexp are re.compile'd. @@ -98,7 +97,7 @@ def new(obj: MutableMapping, path: Path, value, separator="/", creator: Creator return segments.set(obj, split_segments, value) -def delete(obj: MutableMapping, glob: GlobExtend, separator="/", afilter: Filter = None) -> int: +def delete(obj: MutableMapping, glob: Glob, separator="/", afilter: Filter = None) -> int: """ Given a obj, delete all elements that match the glob. @@ -157,7 +156,7 @@ def f(obj, pair, counter): return deleted -def set(obj: MutableMapping, glob: GlobExtend, value, separator="/", afilter: Filter = None) -> int: +def set(obj: MutableMapping, glob: Glob, value, separator="/", afilter: Filter = None) -> int: """ Given a path glob, set all existing elements in the document to the given value. Returns the number of elements changed. @@ -184,7 +183,7 @@ def f(obj, pair, counter): def get( obj: MutableMapping, - glob: GlobExtend, + glob: Glob, separator="/", default: Any = _DEFAULT_SENTINEL ) -> Union[MutableMapping, object, Callable]: @@ -223,7 +222,7 @@ def f(_, pair, results): return results[0] -def values(obj: MutableMapping, glob: GlobExtend, separator="/", afilter: Filter = None, dirs=True): +def values(obj: MutableMapping, glob: Glob, separator="/", afilter: Filter = None, dirs=True): """ Given an object and a path glob, return an array of all values which match the glob. The arguments to this function are identical to those of search(). @@ -233,7 +232,7 @@ def values(obj: MutableMapping, glob: GlobExtend, separator="/", afilter: Filter return [v for p, v in search(obj, glob, yielded, separator, afilter, dirs)] -def search(obj: MutableMapping, glob: GlobExtend, yielded=False, separator="/", afilter: Filter = None, dirs=True): +def search(obj: MutableMapping, glob: Glob, yielded=False, separator="/", afilter: Filter = None, dirs=True): """ Given a path glob, return a dictionary containing all keys that matched the given glob. diff --git a/dpath/types.py b/dpath/types.py index fc9c68e..01dd2af 100644 --- a/dpath/types.py +++ b/dpath/types.py @@ -47,7 +47,7 @@ class MergeType(IntFlag): replaces the destination in this situation.""" -PathSegment = Union[int, str, bytes] +PathSegment = Union[int, str, bytes, Pattern] """Type alias for dict path segments where integers are explicitly casted.""" Filter = Callable[[Any], bool] @@ -55,23 +55,11 @@ class MergeType(IntFlag): (Any) -> bool""" -Glob = Union[str, Sequence[str]] +Glob = Union[str, Pattern, Sequence[Union[str, Pattern]]] """Type alias for glob parameters.""" -GlobExtend = Union[str, Pattern, Sequence[Union[str, Pattern]]] -""" -Type alias for globs extended by the possibility to have re.Pattern. -Note that this is a subset of Path, but it seems that changing -typing from Glob to GlobExtend is sufficient. - -Main reason for allowing Pattern is to treat similarly the path '{regex}' and -re.compile("regex"). - -This is tested in tests.test_various_exts. -""" - -Path = Union[str, Sequence[PathSegment]] +Path = Union[str, Pattern, Sequence[PathSegment]] """Type alias for path parameters.""" Hints = Sequence[Tuple[PathSegment, type]] From d6ba501fea75d6fa4118a0bf7183ceace01a2d71 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 16 Apr 2023 01:52:07 +0300 Subject: [PATCH 10/16] Wording --- dpath/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dpath/__init__.py b/dpath/__init__.py index 6361695..cef028e 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -33,8 +33,7 @@ def _split_path(path: Glob, separator: Optional[str] = "/") -> Union[List[PathSegment], PathSegment]: """ - Given a path and separator, return a tuple of segments; string segments that represent - (i.e. "{.*}") regexp are re.compile'd. + Given a path and separator, return a tuple of segments. If path is already a non-leaf thing, return it: this covers sequences of strings and re.Patterns. @@ -44,7 +43,9 @@ def _split_path(path: Glob, separator: Optional[str] = "/") -> Union[List[PathSe ignored, and is assumed to be part of each key glob. It will not be stripped (i.e. a first list element can be an empty string). - Errors in re.compilation raise InvalidRegex exception. + If RegEx support is enabled then str segments which are wrapped with curly braces will be handled as regular + expressions. These segments will be compiled using re.compile. + Errors during RegEx compilation will raise an InvalidRegex exception. """ # First split the path into segments, validate wrt. type annotation GlobExtend if not segments.leaf(path): From 4d0324d03c6882d1de25dcd920cb1de8ca727aeb Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 16 Apr 2023 01:52:47 +0300 Subject: [PATCH 11/16] Isolation of regex handling code --- dpath/__init__.py | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/dpath/__init__.py b/dpath/__init__.py index cef028e..5c14da5 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -47,35 +47,29 @@ def _split_path(path: Glob, separator: Optional[str] = "/") -> Union[List[PathSe expressions. These segments will be compiled using re.compile. Errors during RegEx compilation will raise an InvalidRegex exception. """ - # First split the path into segments, validate wrt. type annotation GlobExtend if not segments.leaf(path): split_segments = path elif isinstance(path, re.Pattern): - # Allow a path which is a single re.Pattern + # Handle paths which are comprised of a single re.Pattern split_segments = (path,) - elif isinstance(path, (int, float, bool, type(None))): - # This protects against situations that are not screened by segment.leaf. These should - # not occur with spec. "path:GlobExtend", but type annotations do not check arguments. - # Thus the error message than is clearer than "...object has no attribute lstrip" - raise ValueError(f"Error: scalars cannot appear outside of sequence in {path}") else: split_segments = path.lstrip(separator).split(separator) - # now we re.compile segments that represent re.Regexps. - split_compiled_segments = [] - for segment in split_segments: - if options.DPATH_ACCEPT_RE_REGEXP_IN_STRING and isinstance(segment, str) and segment[0] == "{" and segment[-1] == "}": - try: - rs = segment[1:-1] - rex = re.compile(rs) - except re.error as reErr: - raise InvalidRegex(f"In segment '{segment}' string '{rs}' not accepted" - + f" as re.regexp:\n==>\t{reErr}") - split_compiled_segments.append(rex) - else: - split_compiled_segments.append(segment) - - return split_compiled_segments + if options.DPATH_ACCEPT_RE_REGEXP_IN_STRING: + # Handle RegEx segments + + def compile_regex_segment(segment: PathSegment): + if isinstance(segment, str) and len(reg := segment.removeprefix("{").removesuffix("}")) == len(segment) - 2: + try: + return re.compile(reg) + except re.error as re_err: + raise InvalidRegex(f"Could not compile RegEx in path segment '{reg}' ({re_err})") + + return segment + + split_segments = list(map(compile_regex_segment, split_segments)) + + return split_segments def new(obj: MutableMapping, path: Path, value, separator="/", creator: Creator = None) -> MutableMapping: From 659824f978837d4c41aa1bc3f4a5900a85233b76 Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Sun, 16 Apr 2023 02:05:29 +0300 Subject: [PATCH 12/16] Remove unsupported str functions --- dpath/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dpath/__init__.py b/dpath/__init__.py index 5c14da5..5507c4d 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -59,11 +59,11 @@ def _split_path(path: Glob, separator: Optional[str] = "/") -> Union[List[PathSe # Handle RegEx segments def compile_regex_segment(segment: PathSegment): - if isinstance(segment, str) and len(reg := segment.removeprefix("{").removesuffix("}")) == len(segment) - 2: + if isinstance(segment, str) and segment.startswith("{") and segment.endswith("}"): try: - return re.compile(reg) + return re.compile(segment[1:-1]) except re.error as re_err: - raise InvalidRegex(f"Could not compile RegEx in path segment '{reg}' ({re_err})") + raise InvalidRegex(f"Could not compile RegEx in path segment '{segment}' ({re_err})") return segment From 6d54bd97f6a6375b6ce8a87240de17bdb99639ea Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Wed, 19 Apr 2023 03:14:21 +0300 Subject: [PATCH 13/16] Rename feature flag and simplify envar code --- README.rst | 6 +++++- dpath/__init__.py | 2 +- dpath/options.py | 21 +-------------------- tests/test_various_exts.py | 4 ++-- 4 files changed, 9 insertions(+), 24 deletions(-) diff --git a/README.rst b/README.rst index fbfdc22..19cdfbd 100644 --- a/README.rst +++ b/README.rst @@ -462,7 +462,11 @@ Python's `re` regular expressions PythonRe_ may be used as follows: >>> # enable >>> dpath.options.DPATH_ACCEPT_RE_REGEXP_IN_STRING = True >>> # disable - >>> dpath.options.DPATH_ACCEPT_RE_REGEXP_IN_STRING = False + >>> dpath.options.ALLOW_REGEX = False + + - Now a path component may also be specified : + + - in a path expression, as {} where - Now a path component may also be specified : diff --git a/dpath/__init__.py b/dpath/__init__.py index 5507c4d..23da12c 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -55,7 +55,7 @@ def _split_path(path: Glob, separator: Optional[str] = "/") -> Union[List[PathSe else: split_segments = path.lstrip(separator).split(separator) - if options.DPATH_ACCEPT_RE_REGEXP_IN_STRING: + if options.ALLOW_REGEX: # Handle RegEx segments def compile_regex_segment(segment: PathSegment): diff --git a/dpath/options.py b/dpath/options.py index fe2a34c..f33cccf 100644 --- a/dpath/options.py +++ b/dpath/options.py @@ -2,29 +2,10 @@ ALLOW_EMPTY_STRING_KEYS = False -DPATH_ACCEPT_RE_REGEXP_IN_STRING = False +ALLOW_REGEX = "DPATH_ALLOW_REGEX" in environ """Enables regular expression support. Enabling this feature will allow usage of regular expressions as part of paths. Regular expressions must be wrapped in curly brackets. For example: "a/b/{[cd]}". Expressions will be compiled using the standard library re.compile function. """ - -# ---------------------------------------------------------------------------------------------- -# undocumented feature for testability: facilitate running the entire package (or test suite) -# with option enabled/disabled by setting "DPATH_ACCEPT_RE_REGEXP_IN_STRING" in process environment. -# Value => Effect -# TRUE : set to True -# TRUE_PRINT: set to True and output confirmation to stderr -# FALSE: set to False -# FALSE_PRINT: set to False and output confirmation to stderr -if "DPATH_ACCEPT_RE_REGEXP_IN_STRING" in environ: - setTrue = environ["DPATH_ACCEPT_RE_REGEXP_IN_STRING"] - if setTrue in ("TRUE", "TRUE_PRINT"): - DPATH_ACCEPT_RE_REGEXP_IN_STRING = True - else: - DPATH_ACCEPT_RE_REGEXP_IN_STRING = False - - if setTrue[-5:] == "PRINT": - from sys import stderr - print(f"DPATH_ACCEPT_RE_REGEXP_IN_STRING={DPATH_ACCEPT_RE_REGEXP_IN_STRING}", file=stderr) diff --git a/tests/test_various_exts.py b/tests/test_various_exts.py index a5ff8b1..4fc2b72 100644 --- a/tests/test_various_exts.py +++ b/tests/test_various_exts.py @@ -17,7 +17,7 @@ from dpath.exceptions import InvalidRegex # check that how the options have been set -print(f"At entry in test_path_ext DPATH_ACCEPT_RE_REGEXP_IN_STRING = {DP.options.DPATH_ACCEPT_RE_REGEXP_IN_STRING}", file=sys.stderr) +print(f"At entry in test_path_ext DPATH_ACCEPT_RE_REGEXP_IN_STRING = {DP.options.ALLOW_REGEX}", file=sys.stderr) class SampleDicts(): @@ -276,7 +276,7 @@ def test1(self): # Now tests that require DP.options.DPATH_ACCEPT_RE_REGEXP_IN_STRING to be set -if not DP.options.DPATH_ACCEPT_RE_REGEXP_IN_STRING: +if not DP.options.ALLOW_REGEX: print("Skipping because DPATH_ACCEPT_RE_REGEXP_IN_STRING != True", file=sys.stderr) else: From 4c806d18b82db804e4f0ba62eac12918f8a99c0a Mon Sep 17 00:00:00 2001 From: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Wed, 19 Apr 2023 11:20:55 +0300 Subject: [PATCH 14/16] Run tests with regex feature flag using workflow --- .github/workflows/tests.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ba8e2bf..428d953 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -87,3 +87,10 @@ jobs: uses: ymyzk/run-tox-gh-actions@main with: tox-args: -vv --hashseed=${{ needs.generate-hashseed.outputs.hashseed }} + + - name: Run tox with tox-gh-actions (Regex feature flag) + uses: ymyzk/run-tox-gh-actions@main + env: + ALLOW_REGEX: True + with: + tox-args: -vv --hashseed=${{ needs.generate-hashseed.outputs.hashseed }} From 1b6143d07cb58def7d784a3334706e13383984b7 Mon Sep 17 00:00:00 2001 From: Alain Lich Date: Wed, 19 Apr 2023 15:22:10 +0200 Subject: [PATCH 15/16] Tests for regexp feature redone, files renamed. **NOTE** this is for my testing; if all goes well I will suppress extra workflow and push again --- .github/workflows/python3Test.yml | 8 +- .github/workflows/tests.yml | 2 +- README.rst | 6 +- tests/regexpTestLib.py | 179 +++++++++++++++ tests/test_regexp.py | 334 +++++++++++++++++++++++++++ tests/test_various_exts.py | 359 ------------------------------ tox-set-rex.ini | 4 +- 7 files changed, 521 insertions(+), 371 deletions(-) create mode 100644 tests/regexpTestLib.py create mode 100644 tests/test_regexp.py delete mode 100644 tests/test_various_exts.py diff --git a/.github/workflows/python3Test.yml b/.github/workflows/python3Test.yml index 1a5a67a..25a3a23 100644 --- a/.github/workflows/python3Test.yml +++ b/.github/workflows/python3Test.yml @@ -68,23 +68,23 @@ jobs: pip install tox echo "Installed tox" - - name: Tox test with default DPATH_ACCEPT_RE_REGEXP_IN_STRING = FALSE + - name: Tox test with default DPATH_ALLOW_REGEX not set shell: bash if: always() # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ # tox testing, here tox.ini is used - # DPATH_ACCEPT_RE_REGEXP_IN_STRING = FALSE (default) + # DPATH_ALLOW_REGEX not set # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ run: | tox echo "Ran tox" - - name: Tox test with DPATH_ACCEPT_RE_REGEXP_IN_STRING = TRUE + - name: Tox test with DPATH_ALLOW_REGEX = TRUE shell: bash if: always() # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ # tox testing, here tox-set-rex.ini is used - # DPATH_ACCEPT_RE_REGEXP_IN_STRING = TRUE + # DPATH_ALLOW_REGEX = TRUE # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ run: | tox -c tox-set-rex.ini diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 428d953..35261d6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -91,6 +91,6 @@ jobs: - name: Run tox with tox-gh-actions (Regex feature flag) uses: ymyzk/run-tox-gh-actions@main env: - ALLOW_REGEX: True + DPATH_ALLOW_REGEX: True with: tox-args: -vv --hashseed=${{ needs.generate-hashseed.outputs.hashseed }} diff --git a/README.rst b/README.rst index 19cdfbd..1f37fa0 100644 --- a/README.rst +++ b/README.rst @@ -460,14 +460,10 @@ Python's `re` regular expressions PythonRe_ may be used as follows: >>> import dpath >>> # enable - >>> dpath.options.DPATH_ACCEPT_RE_REGEXP_IN_STRING = True + >>> dpath.options.ALLOW_REGEX = True >>> # disable >>> dpath.options.ALLOW_REGEX = False - - Now a path component may also be specified : - - - in a path expression, as {} where - - Now a path component may also be specified : - in a path expression, as {} where `` is a regular expression diff --git a/tests/regexpTestLib.py b/tests/regexpTestLib.py new file mode 100644 index 0000000..0778938 --- /dev/null +++ b/tests/regexpTestLib.py @@ -0,0 +1,179 @@ +# -*- coding: utf-8 -*- +# -*- mode: Python -*- +# +# (C) Alain Lichnewsky, 2023 +# +# These classes are used to simplify testing code in test_regexp_exts_simple.py. +# +# They allow: +# 1) to iterate over a sequence of test-cases specifying: +# a) a glob expression describing a list of paths +# b) a list of expected results +# 2) apply a function on each test-case +# 3) check that all outputs are withing the specified results and that all +# expected results have been produced at least once +# +# The currently implemented classes: do not check the order of the output results, +# or the multiplicity (beyond "at least once") +# +# Examples are shown in file test_regexp_ext_simple.py. +# ------------------------------------------------------------------------ + +import sys + +from typing import Sequence, Union, Dict +from pprint import PrettyPrinter + +# single point of parametrization of output stream +_print_file = sys.stderr + + +def show(*posArgs, **keyArgs): + print(*posArgs, file=_print_file, **keyArgs) + + +class Loop: + """ Given a dict and a specification table (containing a path/glob specification + and a list of expected result), apply a function to each spec and verify the + result wrt. the expected result. + + The run method checks that all results are in the expected list and that each + expected result has been produced once. No consideration is given to multiplicity and order. + """ + def __init__(self, data: Union[Sequence, Dict], specs: Sequence): + """ Defines the data dict/sequence to which functions are applied, and + a sequence of test cases specified with tuples of an input and a sequence of outputs. + + Args: + data (Union[Sequence, Dict]): the dict to which dpath functions are applied + specs (Sequence): Each entry is a tuple: (test specification, output) + """ + self.data = data + self.specs = specs + self.verbose = False + self.indent = 12 + self.pretty = PrettyPrinter(indent=self.indent, width=120) + self.pp = lambda x: x + + def setVerbose(self, v=True): + """set the verbosity level, if true all tests cases and results are listed + + Args: + v (bool, optional):Defaults to True + + Returns: self, for chaining methods + """ + self.verbose = v + return self + + def setPrettyPrint(self): + """Set pretty printing mode + + Returns: self, for chaining methods + """ + self.pp = self._pretty + return self + + def _pretty(self, x): + """Internal method for returning : + - if PrettyPrint is set: a pretty printed/indented result + - otherwise : the unchanged input + + Args: + x (Any): object which can be processed by Python's pretty printer + + Returns: a pretty string + """ + def do_NL(x): + if "\n" in x: + return "\n" + " " * self.indent + x + else: + return x + + return do_NL(self.pretty.pformat(x)) + + def _validate_collect(self, result, expected, found): + """ (internal) Checks that the result produced is in the 'expected' field of the test + specification table. No exception is expected, but the user may have special + result strings to denote exception. (see examples in test_regexp_ext_simple) + + The result is collected for later determination of missing results wrt. expected. + + Args: + result : the result to be tested or an Exception instance + expected : sequence of expected results + found ( set): set used to collect results + """ + if result is None: + assert expected is None + elif expected is None: + show(f"Error: Expected result: None, result={result}") + assert result is None + else: + # this simplifies specs tables when a single output is expected + if isinstance(expected, (dict, bool, str)): + expected = (expected,) + assert result in expected + found.append(result) + + def _validate_collection(self, expected, found, spec, specCount): + """ (internal) Checks that the found sequence covers all expected values. + Args: + expected (Sequence): expected results + found ( Set): observed results + spec (Sequence): dpath parameter (Glob) + specCount (int): position in specification table, printed to facilitate identification + of diagnostics. + """ + if expected is not None: + if isinstance(expected, (dict, bool, str)): + expected = (expected,) + + # compute difference between found and expected + diff = [x for x in expected if (x not in found)] + if len(found) == 0: + found = "None" + + # tell the user + if len(diff) != 0: + if not self.verbose: + show(f"\t{specCount:2} spec:{spec}") + show(f"Error\t(Sets) Found:{self.pp(found)}\n\tExpected:{self.pp(expected)}") + show(f"Expected values missing : {self.pp(diff)}") + assert len(diff) == 0 + else: + if len(found) > 0: + if not self.verbose: + show(f"\t{specCount:2} spec:{self.pp(spec)},\n\t expected={self.pp(expected)},\n\tself.pp(found)") + assert len(found) == 0 + + def run(self, func): + """For each tuple in the specification table, apply function func with + arguments (data, specification) and check that the result is valid. + + If verbose set, outputs test case and sequence of results. + + The set of results of function application is collected and analyzed. + + Args: + func (data, spec) -> result: function called with arguments data and test case + specification, returns result to be monitored. + """ + specCount = 0 + for (spec, expected) in self.specs: + specCount += 1 + + if isinstance(expected, str): + expected = (expected,) + + if self.verbose: + show(f"\t{specCount:2} spec:{self.pp(spec)},\t expected={self.pp(expected)}") + + found = [] + for result, value in func(self.data, spec): + if self.verbose: + show(f"\t\tpath:{result}\tvalue:{self.pp(value)}\texpected:{self.pp(expected)}") + + self._validate_collect(result, expected, found) + + self._validate_collection(expected, found, spec, specCount) diff --git a/tests/test_regexp.py b/tests/test_regexp.py new file mode 100644 index 0000000..62d708e --- /dev/null +++ b/tests/test_regexp.py @@ -0,0 +1,334 @@ +# -*- coding: utf-8 -*- +# -*- mode: Python -*- +# +# (C) Alain Lichnewsky, 2022, 2023 +# +# Test support of extended specs with re.regex in many dpath functions +# +import sys +from os import environ +import re + +from copy import copy + +import unittest +import dpath as DP +from dpath.exceptions import InvalidRegex + +# reusable classes to perform tests on lists of (case, expected result) +import tests.regexpTestLib as T + +# Allow for command line/environment setup of verbose output +# The default is not set. +_verbosity = "VERBOSE_TEST" in environ and environ["VERBOSE_TEST"] == "TRUE" + + +class SampleDicts: + + d1 = { + "a001": { + "b2": { + "c1.2": { + "d.dd": 0, + "e.ee": 1, + "f.f0": 2, + }, + }, + }, + } + + d2 = { + "Name": "bridge", + "Id": "333d22b3724", + "Created": "2022-12-08T09:02:33.360812052+01:00", + "Driver": "bridge", + "EnableIPv6": False, + "IPAM": { + "Driver": "default", + "Options": None, + "Config": + { + "Subnet": "172.17.0.0/16", + "Gateway": "172.17.0.1" + }}, + "ConfigFrom": { + "Network": "172.O.0.0/32"}, + "Containers": { + "199c590e8f13477": { + "Name": "al_dpath", + "MacAddress": "02:34:0a:11:10:22", + "IPv4Address": "172.17.0.2/16", + "IPv6Address": ""}}, + "Labels": {}} + + +specs1_A = (([re.compile(".*")], "a001"), + ([re.compile("[a-z]+$")], None), + (["*", re.compile(".*")], "a001/b2"), + (["*", "*", re.compile(".*")], "a001/b2/c1.2"), + (["*", re.compile("[a-z]+\\d+$")], "a001/b2"), + (["*", re.compile("[a-z]+[.][a-z]+$")], None), + (["**", re.compile(".*")], ("a001", "a001/b2", "a001/b2/c1.2", "a001/b2/c1.2/d.dd", + "a001/b2/c1.2/e.ee", "a001/b2/c1.2/f.f0")), + (["**", re.compile("[a-z]+\\d+$")], ("a001", "a001/b2")), + (["**", re.compile("[a-z]+[.][a-z]+$")], ('a001/b2/c1.2/d.dd', 'a001/b2/c1.2/e.ee'))) + +specs1_B = (([re.compile(".*")], True), + ([re.compile("[a-z]+$")], False), + (["*", re.compile(".*")], False), + (["*", "*", re.compile(".*")], False), + (["*", re.compile("[a-z]+\\d+$")], False), + (["*", re.compile("[a-z]+[.][a-z]+$")], False), + (["**", re.compile(".*")], True), + (["**", re.compile("[a-z]+\\d+$")], True), + (["**", re.compile("[a-z]+[.][a-z]+$")], False)) + +specs1_C = (([re.compile(".*")], set()), + ([re.compile("[a-z]+$")], set()), + (["*", re.compile(".*")], set()), + (["*", "*", re.compile(".*")], set()), + (["*", re.compile("[a-z]+\\d+$")], set()), + (["*", re.compile("[a-z]+[.][a-z]+$")], set()), + (["**", re.compile(".*")], set((0, 1, 2))), + (["**", re.compile("[a-z]+\\d+$")], set()), + (["**", re.compile("[a-z]+[.][a-z]+$")], set((0, 1)))) + +specs1_D = (([re.compile(".*")], None), + ([re.compile("[a-z]+$")], None), + (["*", re.compile(".*")], None), + (["*", "*", re.compile(".*")], None), + (["*", re.compile("[a-z]+\\d+$")], None), + (["*", re.compile("[a-z]+[.][a-z]+$")], None), + (["**", re.compile(".*")], ("a001/b2/c1.2/d.dd", + "a001/b2/c1.2/e.ee", "a001/b2/c1.2/f.f0")), + (["**", re.compile("[a-z]+\\d+$")], None), + (["**", re.compile("[a-z]+[.][a-z]+$")], ('a001/b2/c1.2/d.dd', 'a001/b2/c1.2/e.ee'))) + +specs1_View = (([re.compile(".*")], ({'a001': {'b2': {'c1.2': {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}}}},)), + ([re.compile("[a-z]+$")], ({},)), + (["*", re.compile(".*")], ({'a001': {'b2': {'c1.2': {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}}}},)), + (["*", "*", re.compile(".*")], ({'a001': {'b2': {'c1.2': {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}}}},)), + (["*", re.compile("[a-z]+\\d+$")], ({'a001': {'b2': {'c1.2': {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}}}},)), + (["*", re.compile("[a-z]+[.][a-z]+$")], ({},)), + (["**", re.compile(".*")], ({'a001': {'b2': {'c1.2': {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}}}},)), ) + +specs1_Get = (([re.compile(".*")], {'b2': {'c1.2': {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}}}), + ([re.compile("[a-z]+$")], (('*NONE*',),)), + (["*", re.compile(".*")], {'c1.2': {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}}), + (["*", "*", re.compile(".*")], {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}), + (["*", re.compile("[a-z]+\\d+$")], {'c1.2': {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}}), + (["*", re.compile("[a-z]+[.][a-z]+$")], (('*NONE*',),)), + (["**", re.compile(".*")], ("Exception",)), + (["**", re.compile("[a-z]+\\d+$")], ("Exception",)), + (["**", re.compile("[a-z]+[.][a-z]+$")], ("Exception",)),) + +specs2_Search = ((["*", re.compile("[A-Z][a-z\\d]*$")], + ("IPAM/Driver", "IPAM/Options", "IPAM/Config", "ConfigFrom/Network")), + (["**", re.compile("[A-Z][a-z\\d]*$")], + ("Name", "Id", "Created", "Driver", + "Containers", "Labels", "IPAM/Driver", "IPAM/Options", + "IPAM/Config", "IPAM/Config/Subnet", "IPAM/Config/Gateway", + "ConfigFrom/Network", "Containers/199c590e8f13477/Name")), + (["**", re.compile("[A-Z][A-Za-z\\d]*Address$")], + ("Containers/199c590e8f13477/MacAddress", "Containers/199c590e8f13477/IPv4Address", + "Containers/199c590e8f13477/IPv6Address")), + (["**", re.compile("[A-Za-z]+\\d+$")], ("EnableIPv6",)), + (["**", re.compile("\\d+[.]\\d+")], None), + + # repeated intentionally using raw strings rather than '\\' escapes + + (["*", re.compile(r"[A-Z][a-z\d]*$")], + ("IPAM/Driver", "IPAM/Options", "IPAM/Config", "ConfigFrom/Network")), + (["**", re.compile(r"[A-Za-z]+\d+$")], ("EnableIPv6",)), + (["**", re.compile(r"\d+[.]\d+")], None)) + +specs2_SearchPar = (("**/{^[A-Za-z]{2}$}", ("Id",)), + ("{^[A-Za-z]{2}$}", ("Id",)), + (re.compile("^[A-Za-z]{2}$"), ("Id",)), + ("*/{[A-Z][A-Za-z\\d]*$}", ("IPAM/Driver", "IPAM/Options", "IPAM/Config", "ConfigFrom/Network")), + ("{.*}/{[A-Z][A-Za-z\\d]*$}", ("IPAM/Driver", "IPAM/Options", "IPAM/Config", "ConfigFrom/Network")), + ("**/{[A-Z][A-Za-z\\d]*\\d$}", ("EnableIPv6",)), + ("**/{[A-Z][A-Za-z\\d]*Address$}", ("Containers/199c590e8f13477/MacAddress", + "Containers/199c590e8f13477/IPv4Address", + "Containers/199c590e8f13477/IPv6Address")), + + # repeated intentionally using raw strings rather than '\\' escapes + + (r"**/{[A-Z][A-Za-z\d]*\d$}", ("EnableIPv6",)),) + + +# one class per function to be tested, postpone tests that need +# DP.options.ALLOW_REGEX == True + +class TestSearchAlways(): + def setUp(self): + if "DPATH_ALLOW_REGEX" in environ: + DP.options.ALLOW_REGEX = True + + def test1(self): + T.show(f"In {self.test1}") + tests = T.Loop(SampleDicts.d1, specs1_A) + + def fn(_data, _spec): + return DP.search(_data, _spec, yielded=True) + + tests.setVerbose(_verbosity).run(fn) + + def test2(self): + T.show(f"In {self.test2}") + tests = T.Loop(SampleDicts.d1, specs1_D) + + def afilter(x): + if isinstance(x, int): + return True + return False + + def fn(_data, _spec): + return DP.search(_data, _spec, yielded=True, afilter=afilter) + + tests.setVerbose(_verbosity).run(fn) + + def test3(self): + T.show(f"In {self.test3}") + tests = T.Loop(SampleDicts.d2, specs2_Search) + + def fn(_data, _spec): + return DP.search(_data, _spec, yielded=True) + + tests.setVerbose(_verbosity).setPrettyPrint().run(fn) + + +class TestGet(): + def setUp(self): + if "DPATH_ALLOW_REGEX" in environ: + DP.options.ALLOW_REGEX = True + + def test1(self): + T.show(f"In {self.test1}") + tests = T.Loop(SampleDicts.d1, specs1_Get) + + def fn(_data, _spec): + try: + return ((DP.get(_data, _spec, default=("*NONE*",)), None),) + except InvalidRegex as err: + T.show(f"Exception: {err}") + return (("InvalidRegex", None), ) + except Exception as err: + T.show(f"Exception: {err}") + return (("Exception", None), ) + + tests.setVerbose(_verbosity).run(fn) + + +class TestView(): + def setUp(self): + if "DPATH_ALLOW_REGEX" in environ: + DP.options.ALLOW_REGEX = True + + def test1(self): + T.show(f"In {self.test1}") + tests = T.Loop(SampleDicts.d1, specs1_View) + + def fn(_data, _spec): + r = DP.segments.view(_data, _spec) + return ((r, None), ) + + tests.setVerbose(_verbosity).run(fn) + + +class TestMatch(): + def setUp(self): + if "DPATH_ALLOW_REGEX" in environ: + DP.options.ALLOW_REGEX = True + + def test1(self): + T.show(f"In {self.test1}") + tests = T.Loop(SampleDicts.d1, specs1_B) + + def fn(_data, _spec): + r = DP.segments.match(_data, _spec) + return ((r, None), ) + + tests.setVerbose(_verbosity).run(fn) + + +class TestSearch(): + def setUp(self): + # these tests involve regex in parenthesized strings + if "DPATH_ALLOW_REGEX" in environ: + DP.options.ALLOW_REGEX = True + + if DP.options.ALLOW_REGEX is not True: + DP.options.ALLOW_REGEX = True + T.show("ALLOW_REGEX == True required for this test: forced") + + def test1(self): + T.show(f"In {self.test1}") + tests = T.Loop(SampleDicts.d2, specs2_SearchPar) + + def fn(_data, _spec): + return DP.search(_data, _spec, yielded=True) + + tests.setVerbose(_verbosity).setPrettyPrint().run(fn) + + def test2(self): + T.show(f"In {self.test1}") + specs = (("/**/{zz)bad}", ("InvalidRegex",)), + ("{zz)bad}/yyy", ("InvalidRegex",)), + ("**/{zz)bad}/yyy", ("InvalidRegex",)), + ("**/{zz)bad}/yyy/.*", ("InvalidRegex",)), + (123, ("Exception",))) + + tests = T.Loop(SampleDicts.d2, specs) + + def fn(_data, _spec): + try: + return DP.search(_data, _spec, yielded=True) + except InvalidRegex as err: + if tests.verbose: + T.show(f"\tErrMsg: {err}") + return (("InvalidRegex", None),) + except Exception as err: + if tests.verbose: + T.show(f"\tErrMsg: {err}") + return (("Exception", None),) + + tests.setVerbose(_verbosity).setPrettyPrint().run(fn) + + +class TestDelete(unittest.TestCase): + def setUp(self): + # these tests involve regex in parenthesized strings + if "DPATH_ALLOW_REGEX" in environ: + DP.options.ALLOW_REGEX = True + + if DP.options.ALLOW_REGEX is not True: + DP.options.ALLOW_REGEX = True + T.show("ALLOW_REGEX == True required for this test: forced") + + def test1(self): + T.show(f"In {self.test1}") + dict1 = { + "a": { + "b": 0, + "12": 0, + }, + "a0": { + "b": 0, + }, + } + + specs = (re.compile("[a-z]+$"), re.compile("[a-z]+\\d+$"), + "{[a-z]+\\d+$}") + i = 0 + for spec in specs: + dict = copy(dict1) + print(f"spec={spec}") + print(f"Before deletion dict={dict}", file=sys.stderr) + DP.delete(dict, [spec]) + print(f"After deletion dict={dict}", file=sys.stderr) + if i == 0: + assert (dict == {"a0": {"b": 0, }, }) + else: + assert (dict == {"a": {"b": 0, "12": 0, }}) + i += 1 diff --git a/tests/test_various_exts.py b/tests/test_various_exts.py deleted file mode 100644 index 4fc2b72..0000000 --- a/tests/test_various_exts.py +++ /dev/null @@ -1,359 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -*- mode: Python -*- -# -# (C) Alain Lichnewsky, 2022, 2023 -# -# Test support of extended specs with re.regex in many functionalities that use path -# specifications. -# -import sys -import re - -from copy import copy - -import unittest -import dpath as DP -from dpath.exceptions import InvalidRegex - -# check that how the options have been set -print(f"At entry in test_path_ext DPATH_ACCEPT_RE_REGEXP_IN_STRING = {DP.options.ALLOW_REGEX}", file=sys.stderr) - - -class SampleDicts(): - - def build(self): - self.d1 = { - "a001": { - "b2": { - "c1.2": { - "d.dd": 0, - "e.ee": 1, - "f.f0": 2, - }, - }, - }, - } - - self.specs1 = (([re.compile(".*")], "a001", True, set()), - ([re.compile("[a-z]+$")], "a001", False, set()), - (["*", re.compile(".*")], "a001", False, set()), - (["*", "*", re.compile(".*")], "a001", False, set()), - (["*", re.compile("[a-z]+\\d+$")], "a001", False, set()), - (["*", re.compile("[a-z]+[.][a-z]+$")], "a001", False, set()), - (["**", re.compile(".*")], "a001", True, set((0, 1, 2))), - (["**", re.compile("[a-z]+\\d+$")], "a001", True, set()), - (["**", re.compile("[a-z]+[.][a-z]+$")], "a001", False, set((0, 1)))) - - self.specs1Pairs = (([re.compile(".*")], ("a001",)), - ([re.compile("[a-z]+$")], None), - (["*", re.compile(".*")], ("a001/b2",)), - (["*", "*", re.compile(".*")], ("a001/b2/c1.2",)), - (["*", re.compile("[a-z]+\\d+$")], ("a001/b2",)), - (["*", re.compile("[a-z]+[.][a-z]+$")], None), - (["**", re.compile(".*")], - ("a001", "a001/b2", "a001/b2/c1.2", "a001/b2/c1.2/d.dd", - "a001/b2/c1.2/e.ee", "a001/b2/c1.2/f.f0")), - (["**", re.compile("[a-z]+\\d+$")], ("a001", "a001/b2")), - (["**", re.compile("[a-z]+[.][a-z]+$")], - ("a001/b2/c1.2/d.dd", "a001/b2/c1.2/e.ee")) - ) - - self.specs1GetPairs = (([re.compile(".*")], {'b2': {'c1.2': {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}}}), - ([re.compile("[a-z]+$")], ('*NONE*',)), - (["*", re.compile(".*")], {'c1.2': {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}}), - (["*", "*", re.compile(".*")], {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}), - (["*", re.compile("[a-z]+\\d+$")], {'c1.2': {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}}), - (["*", re.compile("[a-z]+[.][a-z]+$")], ('*NONE*',)), - (["**", re.compile(".*")], "*FAIL*"), - (["**", re.compile("[a-z]+\\d+$")], "*FAIL*"), - (["**", re.compile("[a-z]+[.][a-z]+$")], "*FAIL*"),) - - self.d2 = { - "Name": "bridge", - "Id": "333d22b3724", - "Created": "2022-12-08T09:02:33.360812052+01:00", - "Scope": "local", - "Driver": "bridge", - "EnableIPv6": False, - "IPAM": { - "Driver": "default", - "Options": None, - "Config": - { - "Subnet": "172.17.0.0/16", - "Gateway": "172.17.0.1" - }}, - "Internal": False, - "Attachable": False, - "Ingress": False, - "ConfigFrom": { - "Network": ""}, - "ConfigOnly": False, - "Containers": { - "199c590e8f13477": { - "Name": "al_dpath", - "EndpointID": "3042bbe16160a63b7", - "MacAddress": "02:34:0a:11:10:22", - "IPv4Address": "172.17.0.2/16", - "IPv6Address": ""}}, - "Options": { - "com.docker.network.bridge.default_bridge": "true", - "com.docker.network.bridge.enable_icc": "true", - "com.docker.network.bridge.enable_ip_masquerade": "true", - "com.docker.network.bridge.host_binding_ipv4": "0.0.0.0", - "com.docker.network.bridge.name": "docker0", - "com.docker.network.driver.mtu": "1500"}, - "Labels": {}} - - self.specs2Pairs = ((["*", re.compile("[A-Z][a-z\\d]*$")], - ("IPAM/Driver", "IPAM/Options", "IPAM/Config", "ConfigFrom/Network")), - (["**", re.compile("[A-Z][a-z\\d]*$")], - ("Name", "Id", "Created", "Scope", "Driver", "Internal", "Attachable", - "Ingress", "Containers", "Options", "Labels", "IPAM/Driver", "IPAM/Options", - "IPAM/Config", "IPAM/Config/Subnet", "IPAM/Config/Gateway", - "ConfigFrom/Network", "Containers/199c590e8f13477/Name")), - (["**", re.compile("[A-Z][A-Za-z\\d]*Address$")], - ("Containers/199c590e8f13477/MacAddress", "Containers/199c590e8f13477/IPv4Address", - "Containers/199c590e8f13477/IPv6Address")), - (["**", re.compile("[A-Za-z]+\\d+$")], ("EnableIPv6",)), - (["**", re.compile("\\d+[.]\\d+")], None), - - # repeated intentionally using raw strings rather than '\\' escapes - - (["*", re.compile(r"[A-Z][a-z\d]*$")], - ("IPAM/Driver", "IPAM/Options", "IPAM/Config", "ConfigFrom/Network")), - (["**", re.compile(r"[A-Z][a-z\d]*$")], - ("Name", "Id", "Created", "Scope", "Driver", "Internal", "Attachable", - "Ingress", "Containers", "Options", "Labels", "IPAM/Driver", "IPAM/Options", - "IPAM/Config", "IPAM/Config/Subnet", "IPAM/Config/Gateway", - "ConfigFrom/Network", "Containers/199c590e8f13477/Name")), - (["**", re.compile(r"[A-Z][A-Za-z\d]*Address$")], - ("Containers/199c590e8f13477/MacAddress", "Containers/199c590e8f13477/IPv4Address", - "Containers/199c590e8f13477/IPv6Address")), - (["**", re.compile(r"[A-Za-z]+\d+$")], ("EnableIPv6",)), - (["**", re.compile(r"\d+[.]\d+")], None) - ) - - self.specs3Pairs = (("**/{^[A-Za-z]{2}$}", ("Id",)), - ("{^[A-Za-z]{2}$}", ("Id",)), - (re.compile("^[A-Za-z]{2}$"), ("Id",)), - ("*/{[A-Z][A-Za-z\\d]*$}", ("IPAM/Driver", "IPAM/Options", "IPAM/Config", "ConfigFrom/Network")), - ("**/{[A-Z][A-Za-z\\d]*\\d$}", ("EnableIPv6",)), - ("**/{[A-Z][A-Za-z\\d]*Address$}", ("Containers/199c590e8f13477/MacAddress", - "Containers/199c590e8f13477/IPv4Address", - "Containers/199c590e8f13477/IPv6Address")), - - # repeated intentionally using raw strings rather than '\\' escapes - - (r"**/{[A-Z][A-Za-z\d]*\d$}", ("EnableIPv6",)), - (r"**/{[A-Z][A-Za-z\d]*Address$}", ("Containers/199c590e8f13477/MacAddress", - "Containers/199c590e8f13477/IPv4Address", - "Containers/199c590e8f13477/IPv6Address")), - ) - return self - - -# one class per function to be tested, postpone tests that need -# DP.options.DPATH_ACCEPT_RE_REGEXP_IN_STRING == True - - -class TestSearchAlways(unittest.TestCase): - def test1(self): - print("Entered test1", file=sys.stderr) - dicts = SampleDicts().build() - dict1 = dicts.d1 - specs = dicts.specs1Pairs - for (spec, expect) in specs: - print(f"Spec={spec}", file=sys.stderr) - found = set() - for (path, value) in DP.search(dict1, spec, yielded=True): - print(f"\tpath={path}\tv={value}\n\texpected={expect}", - file=sys.stderr) - if path is None: - assert expect is None - else: - found.add(path) - assert (path in expect) - if expect is not None: - diff = found ^ set(expect) - if len(diff) != 0: - print(f"Error\tFound:{found}\n\tExpected:{expect}", file=sys.stderr) - print(f"Symmetric Difference : {diff}", file=sys.stderr) - assert False - else: - assert len(found) == 0 - - def test2(self): - print("Entered test2", file=sys.stderr) - print("Test for filtering for int values", file=sys.stderr) - - def afilter(x): - if isinstance(x, int): - return True - return False - - dicts = SampleDicts().build() - dict1 = dicts.d1 - specs = dicts.specs1 - for spec, expected in ((s[0], s[3]) for s in specs): - print(f"Spec={spec}, Key={expected}", file=sys.stderr) - result = set() - for ret in DP.search(dict1, spec, yielded=True, afilter=afilter): - print(f"\tret={ret}", file=sys.stderr) - result.add(ret[1]) - assert isinstance(ret[1], int) - assert result == expected - - def test3(self): - print("Entered test3", file=sys.stderr) - dicts = SampleDicts().build() - dict1 = dicts.d2 - specs = dicts.specs2Pairs - for (spec, expect) in specs: - print(f"Spec={spec}", file=sys.stderr) - found = set() - for (path, value) in DP.search(dict1, spec, yielded=True): - print(f"\tpath={path}\tv={value}", file=sys.stderr) - if path is None: - assert expect is None - else: - found.add(path) - assert (path in expect) - if expect is not None: - diff = found ^ set(expect) - if len(diff) != 0: - print(f"Error\tFound:{found}\n\tExpected:{expect}", file=sys.stderr) - print(f"Symmetric Difference: {diff}", file=sys.stderr) - assert False - else: - assert len(found) == 0 - - -class TestGet(unittest.TestCase): - - def test1(self): - print("Entered test1", file=sys.stderr) - - dicts = SampleDicts().build() - dict1 = dicts.d1 - specs = dicts.specs1GetPairs - for (spec, expect) in specs: - print(f"Spec={spec}", file=sys.stderr) - try: - ret = DP.get(dict1, spec, default=("*NONE*",)) - print(f"\tret={ret}", file=sys.stderr) - assert (ret == expect) - except ValueError as err: - print("\t get fails:", err, type(err), file=sys.stderr) - assert (expect == "*FAIL*") - - -class TestView(unittest.TestCase): - def test1(self): - print("Entered test1", file=sys.stderr) - - dicts = SampleDicts().build() - dict1 = dicts.d1 - specs = dicts.specs1 - for spec in specs: - for ret in DP.segments.view(dict1, spec[0]): - print(f"\tview spec:{spec} returns:{ret}", file=sys.stderr) - assert ret == spec[1] - - -class TestMatch(unittest.TestCase): - def test1(self): - print("Entered test1", file=sys.stderr) - - dicts = SampleDicts().build() - dict1 = dicts.d1 - specs = dicts.specs1 - for spec in specs: - ret = DP.segments.match(dict1, spec[0]) - print(f"\tmatch spec:{spec} returns:{ret}", file=sys.stderr) - assert ret == spec[2] - - -# Now tests that require DP.options.DPATH_ACCEPT_RE_REGEXP_IN_STRING to be set -if not DP.options.ALLOW_REGEX: - print("Skipping because DPATH_ACCEPT_RE_REGEXP_IN_STRING != True", file=sys.stderr) - -else: - - class TestSearch(unittest.TestCase): - - def test4(self): - print("Entered test4", file=sys.stderr) - dicts = SampleDicts().build() - dict1 = dicts.d2 - specs = dicts.specs3Pairs - for (spec, expect) in specs: - print(f"Spec={spec}", file=sys.stderr) - found = set() - for (path, value) in DP.search(dict1, spec, yielded=True): - print(f"\tpath={path}\tv={value}", file=sys.stderr) - if path is None: - assert expect is None - else: - found.add(path) - assert (path in expect) - if expect is not None: - diff = found ^ set(expect) - if len(diff) != 0: - print(f"Error\tFound:{found}\n\tExpected:{expect}", file=sys.stderr) - print(f"Symmetric Difference : {diff}", file=sys.stderr) - assert False - else: - assert len(found) == 0 - - def test5(self): - print("Entered test5 -- re.error::", file=sys.stderr) - dicts = SampleDicts().build() - dict1 = dicts.d2 - specs = (("/**/{zz)bad}", "ERROR"), - ("{zz)bad}/yyy", "ERROR"), - ("**/{zz)bad}/yyy", "ERROR"), - ("**/{zz)bad}/yyy/.*", "ERROR"), - (123, "OTHERERROR")) - - for (spec, expect) in specs: - print(f"Spec={spec}", file=sys.stderr) - try: - for (path, value) in DP.search(dict1, spec, yielded=True): - print(f"\tpath={path}\tv={value}", file=sys.stderr) - assert expect not in ("ERROR", "OTHERERROR") - except InvalidRegex as errExpected: - print(f"Expected error:{errExpected}", file=sys.stderr) - assert expect == "ERROR" - except Exception as errExpected: - print(f"Expected error:{errExpected}", file=sys.stderr) - assert expect == "OTHERERROR" - - class TestDelete(unittest.TestCase): - def test1(self): - print("This is test1", file=sys.stderr) - dict1 = { - "a": { - "b": 0, - "12": 0, - }, - "a0": { - "b": 0, - }, - } - - specs = (re.compile("[a-z]+$"), re.compile("[a-z]+\\d+$"), - "{[a-z]+\\d+$}") - i = 0 - for spec in specs: - dict = copy(dict1) - print(f"spec={spec}") - print(f"Before deletion dict={dict}", file=sys.stderr) - DP.delete(dict, [spec]) - print(f"After deletion dict={dict}", file=sys.stderr) - if i == 0: - assert (dict == {"a0": {"b": 0, }, }) - else: - assert (dict == {"a": {"b": 0, "12": 0, }}) - i += 1 diff --git a/tox-set-rex.ini b/tox-set-rex.ini index 4eca5fe..58819c7 100644 --- a/tox-set-rex.ini +++ b/tox-set-rex.ini @@ -1,4 +1,4 @@ -# ******** THIS VERSION ENABLES DPATH_ACCEPT_RE_REGEXP_IN_STRING +# ******** THIS VERSION ENABLES ALLOW_REGEX # ******** # Tox (http://tox.testrun.org/) is a tool for running tests @@ -27,5 +27,5 @@ deps = nose2 commands = nose2 {posargs} setenv = - DPATH_ACCEPT_RE_REGEXP_IN_STRING = TRUE + DPATH_ALLOW_REGEX = TRUE From 1cfa48e91cd59adc668f9dfaae7d15b9fc398d9b Mon Sep 17 00:00:00 2001 From: Alain Lich Date: Wed, 19 Apr 2023 16:14:02 +0200 Subject: [PATCH 16/16] Removed unneeded workflow files. This adds a redone test_regexp.py. --- .github/workflows/python3Test.yml | 92 ------------------------------- tox-set-rex.ini | 31 ----------- 2 files changed, 123 deletions(-) delete mode 100644 .github/workflows/python3Test.yml delete mode 100644 tox-set-rex.ini diff --git a/.github/workflows/python3Test.yml b/.github/workflows/python3Test.yml deleted file mode 100644 index 25a3a23..0000000 --- a/.github/workflows/python3Test.yml +++ /dev/null @@ -1,92 +0,0 @@ -name: Test python package dpath-python with regexp extension - # ------------------------------------------------------------ - # (C) Alain Lichnewsky, 2021, 2022, 2023 - # - # For running under Github's Actions - # - # ------------------------------------------------------------ - - # ***************************** - # ADDED FOR TESTING PRIOR TO PR - # REMOVE FROM PR submission - # ***************************** - -on: - workflow_dispatch: - # Allows manual dispatch from the Actions tab - -jobs: - test-python3: - - timeout-minutes: 60 - - runs-on: ubuntu-latest - - strategy: - matrix: - # Match versions specified in tox.ini and tox-set-rex.ini - python-version: ['3.8', '3.11', 'pypy3.7', 'pypy3.9'] - - steps: - - name: Checkout code - uses: actions/checkout@main - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@main - with: - python-version: ${{ matrix.python-version }} - architecture: 'x64' - - - name: Ascertain configuration - # - # Collect information concerning $HOME and the location of - # file(s) loaded from Github/ - run: | - echo Working dir: $(pwd) - echo Files at this location: - ls -ltha - echo HOME: ${HOME} - echo LANG: ${LANG} SHELL: ${SHELL} - which python - echo LD_LIBRARY_PATH: ${LD_LIBRARY_PATH} - echo PYTHONPATH: \'${PYTHONPATH}\' - - - name: Install dependencies - shell: bash - if: always() - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - # requirements install the test framework, which is not - # required by the package in setup.py - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - run: | - python -m pip install --upgrade pip setuptools wheel \ - nose2 hypothesis - if [ -f requirements.txt ]; then - pip install -r requirements.txt; - fi - python setup.py install - pip install tox - echo "Installed tox" - - - name: Tox test with default DPATH_ALLOW_REGEX not set - shell: bash - if: always() - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - # tox testing, here tox.ini is used - # DPATH_ALLOW_REGEX not set - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - run: | - tox - echo "Ran tox" - - - name: Tox test with DPATH_ALLOW_REGEX = TRUE - shell: bash - if: always() - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - # tox testing, here tox-set-rex.ini is used - # DPATH_ALLOW_REGEX = TRUE - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - run: | - tox -c tox-set-rex.ini - echo "Ran tox -c tox-set-rex.ini" - \ No newline at end of file diff --git a/tox-set-rex.ini b/tox-set-rex.ini deleted file mode 100644 index 58819c7..0000000 --- a/tox-set-rex.ini +++ /dev/null @@ -1,31 +0,0 @@ -# ******** THIS VERSION ENABLES ALLOW_REGEX -# ******** - -# Tox (http://tox.testrun.org/) is a tool for running tests -# in multiple virtualenvs. This configuration file will run the -# test suite on all supported python versions. To use it, "pip install tox" -# and then run "tox" from this directory. - - -[flake8] -ignore = E501,E722,W503 - -[tox] -envlist = pypy37, pypy39, py38, py311 - -[gh-actions] -python = - pypy-3.7: pypy37 - pypy-3.9: pypy39 - 3.8: py38 - 3.11: py311 - -[testenv] -deps = - hypothesis - mock - nose2 -commands = nose2 {posargs} -setenv = - DPATH_ALLOW_REGEX = TRUE -