diff --git a/.gitignore b/.gitignore index db4561ea..46f268e8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +*.local.* + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -52,3 +54,5 @@ docs/_build/ # PyBuilder target/ +.idea/ +.eggs/ diff --git a/.travis.yml b/.travis.yml index bfc00450..5d12ed6d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,11 @@ -dist: xenial +dist: noble language: python python: - - 2.7 - - 3.4 - - 3.5 - - 3.6 - - 3.7 - - 3.8 - - 3.9 + - 3.10 + - 3.11 + - 3.12 + - 3.13 + - 3.14 before_install: pip install --upgrade setuptools install: pip install tox tox-travis coveralls before_script: if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then tox -e flake8; fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bf873af..33a10934 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog + +# + + +# Version 0.4.0 +Major overhaul of the library. + +* Upgrade of the build system and directory structure to use pyproject.toml +* Update of the dependencies +* Add support for Python 3.10+ +* Remove support for Python 2.7 + + # Version 0.3.61 * fix(tox): remove old EOLed python 3.x versions, added new python versions (@pierresouchay) [#330] diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..6e8c9ed3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,96 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +pyhocon is a Python implementation of the HOCON (Human-Optimized Config Object Notation) parser. It parses HOCON configuration files into Python data structures and can convert to JSON, YAML, properties, and HOCON formats. + +HOCON spec: https://github.com/typesafehub/config/blob/master/HOCON.md + +## Commands + +### Testing +```bash +# Run all tests +pytest tests/ + +# Run a specific test file +pytest tests/test_config_parser.py + +# Run a specific test +pytest tests/test_config_parser.py::TestConfigParser::test_parse_simple_value + +# Run with coverage +coverage run --source=pyhocon -m pytest tests/ +coverage report -m +``` + +### Linting +```bash +flake8 pyhocon tests setup.py.old +``` + +### Tox (multi-environment testing) +```bash +tox # Run all environments +tox -e flake8 # Run flake8 only +tox -e py312 # Run tests on Python 3.12 +``` + +### CLI Tool +```bash +# Convert HOCON to JSON +pyhocon -i input.conf -f json -o output.json +cat input.conf | pyhocon -f json + +# Other formats: json, yaml, properties, hocon +# Use -c for compact output (nested single-value dicts as a.b.c = 1) +``` + +## Architecture + +### Core Modules + +- **config_parser.py** - Main parsing engine using pyparsing library + - `ConfigFactory` - Public API for parsing (parse_file, parse_string, parse_URL, from_dict) + - `ConfigParser` - Internal parser with HOCON grammar rules + +- **config_tree.py** - Data structures + - `ConfigTree` - Hierarchical config storage (extends OrderedDict), supports dot notation access (`config['a.b.c']`) + - `ConfigList` - HOCON arrays + - `ConfigValues` - Concatenated values (handles array/string/dict merging) + - `ConfigSubstitution` - Represents `${var}` and `${?var}` substitutions + +- **converter.py** - `HOCONConverter` with to_json, to_yaml, to_properties, to_hocon methods + +- **period_parser.py / period_serializer.py** - Duration parsing (e.g., "5 days", "10 seconds") + +- **tool.py** - CLI entry point + +### Parsing Flow + +1. `ConfigFactory.parse_*()` receives input +2. `ConfigParser.parse()` applies pyparsing grammar rules +3. Produces `ConfigTree`/`ConfigList` with unresolved `ConfigSubstitution` tokens +4. `resolve_substitutions()` replaces `${var}` references from config or environment variables +5. Returns resolved `ConfigTree` + +### Key Features + +- Substitutions: `${key}` (required) and `${?key}` (optional, fallback to env vars) +- Includes: `include "file.conf"`, `include url("http://...")`, `include required(file("..."))`, glob patterns +- Value access: `config['a.b.c']` or `config['a']['b']['c']` or `config.get_string('a.b.c')` +- Type-safe getters: `get_string()`, `get_int()`, `get_float()`, `get_bool()`, `get_list()`, `get_config()` + +## Dependencies + +- **pyparsing** (>=2, <4) - Grammar parsing +- **python-dateutil** (>=2.8.0, optional) - For months/years in duration parsing + +## Test Dependencies + +- pytest +- mock +- python-dateutil +- coveralls (for CI coverage reporting) diff --git a/README.md b/README.md index c37bccb5..d9421bbf 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ It is available on pypi so you can install it as follows: The parsed config can be seen as a nested dictionary (with types automatically inferred) where values can be accessed using normal dictionary getter (e.g., `conf['a']['b']` or using paths like `conf['a.b']`) or via the methods `get`, `get_int` (throws an exception if it is not an int), `get_string`, `get_list`, `get_float`, `get_bool`, `get_config`. + ```python from pyhocon import ConfigFactory @@ -37,7 +38,7 @@ same_host = conf['databases']['mysql.host'] port = conf['databases.mysql.port'] username = conf['databases']['mysql']['username'] password = conf.get_config('databases')['mysql.password'] -password = conf.get('databases.mysql.password', 'default_password') # use default value if key not found +password = conf.get('databases.mysql.password', 'default_password') # use default value if key not found ``` ## Example of HOCON file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..fd8b88b7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,38 @@ +[build-system] +requires = ["setuptools>=80.10.2", "wheel"] + +[project] +name = "pyhocon_up" +version = "0.4.0" +description = "HOCON parser for Python" +readme = "README.md" +license = "Apache-2.0" +authors = [ + {name = "Robert Rapplean", email = "mythobeast@gmail.com"} +] +keywords = ["hocon", "parser"] +classifiers = [ + "Topic :: Software Development :: Libraries :: Python Modules", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +requires-python = ">=3.10" +dependencies = [ + "pyparsing>=3,<4", +] + +[project.optional-dependencies] +duration = ["python-dateutil>=2.8.0"] +test = ["pytest", "pytest-cov", "mock==5.2.0"] + +[project.urls] +Homepage = "http://github.com/mythobeast/pyhocon/" + +[project.scripts] +pyhocon = "pyhocon.tool:main" + diff --git a/setup.cfg b/setup.cfg.old similarity index 92% rename from setup.cfg rename to setup.cfg.old index b0db9692..d16a65ad 100644 --- a/setup.cfg +++ b/setup.cfg.old @@ -1,5 +1,5 @@ [flake8] -ignore = +ignore = E1,W1 max-line-length = 160 statistics = True count = True diff --git a/setup.py b/setup.py.old old mode 100755 new mode 100644 similarity index 89% rename from setup.py rename to setup.py.old index 7127c77b..be20a5be --- a/setup.py +++ b/setup.py.old @@ -36,7 +36,6 @@ def run_tests(self): author_email='francois.dangngoc@gmail.com', url='http://github.com/chimpler/pyhocon/', classifiers=[ - 'License :: OSI Approved :: Apache Software License', 'Topic :: Software Development :: Libraries :: Python Modules', 'Programming Language :: Python', 'Programming Language :: Python :: 2', @@ -54,18 +53,17 @@ def run_tests(self): ], install_requires=[ 'pyparsing~=2.0;python_version<"3.0"', - 'pyparsing>=2,<4;python_version>="3.0"', + 'pyparsing>=3,<4;python_version>="3.0"', ], extras_require={ - 'Duration': ['python-dateutil>=2.8.0'] + 'Duration': ['python-dateutil>=2.8.0'], + 'test': ['pytest', 'mock==3.0.5'] }, - tests_require=['pytest', 'mock==3.0.5'], entry_points={ 'console_scripts': [ 'pyhocon=pyhocon.tool:main' ] }, - test_suite='tests', cmdclass={ 'test': PyTestCommand } diff --git a/pyhocon/__init__.py b/src/pyhocon/__init__.py similarity index 100% rename from pyhocon/__init__.py rename to src/pyhocon/__init__.py diff --git a/pyhocon/config_parser.py b/src/pyhocon/config_parser.py similarity index 88% rename from pyhocon/config_parser.py rename to src/pyhocon/config_parser.py index 4936f1ee..70130b91 100644 --- a/pyhocon/config_parser.py +++ b/src/pyhocon/config_parser.py @@ -5,125 +5,74 @@ import os import re import socket -import sys -import pyparsing from pyparsing import (Forward, Group, Keyword, Literal, Optional, ParserElement, ParseSyntaxException, QuotedString, Regex, SkipTo, StringEnd, Suppress, TokenConverter, Word, ZeroOrMore, alphanums, alphas8bit, col, lineno, - replaceWith) + replace_with) from pyhocon.period_parser import get_period_expr -# Fix deepcopy issue with pyparsing -if sys.version_info >= (3, 8): - def fixed_get_attr(self, item): - if item == '__deepcopy__': - raise AttributeError(item) - try: - return self[item] - except KeyError: - return "" - - - pyparsing.ParseResults.__getattr__ = fixed_get_attr - from pyhocon.config_tree import (ConfigInclude, ConfigList, ConfigQuotedString, - ConfigSubstitution, ConfigTree, - ConfigUnquotedString, ConfigValues, NoneValue) + ConfigSubstitution, ConfigTree, + ConfigUnquotedString, ConfigValues, NoneValue) from pyhocon.exceptions import (ConfigException, ConfigMissingException, - ConfigSubstitutionException) - -use_urllib2 = False -try: - # For Python 3.0 and later - from urllib.request import urlopen - from urllib.error import HTTPError, URLError -except ImportError: # pragma: no cover - # Fall back to Python 2's urllib2 - from urllib2 import urlopen, HTTPError, URLError - - use_urllib2 = True -try: - basestring -except NameError: # pragma: no cover - basestring = str - unicode = str - -if sys.version_info < (3, 5): - def glob(pathname, recursive=False): - if recursive and '**' in pathname: - import warnings - warnings.warn('This version of python (%s) does not support recursive import' % sys.version) - from glob import glob as _glob - return _glob(pathname) -else: - from glob import glob + ConfigSubstitutionException) + +from urllib.request import urlopen +from urllib.error import HTTPError, URLError + +from glob import glob # Fix deprecated warning with 'imp' library and Python 3.4+. # See: https://github.com/chimpler/pyhocon/issues/248 -if sys.version_info >= (3, 4): - import importlib.util - +import importlib.util - def find_package_dirs(name): - spec = importlib.util.find_spec(name) - # When `imp.find_module()` cannot find a package it raises ImportError. - # Here we should simulate it to keep the compatibility with older - # versions. - if not spec: - raise ImportError('No module named {!r}'.format(name)) - return spec.submodule_search_locations -else: - import imp - import importlib +def find_package_dirs(name): + spec = importlib.util.find_spec(name) + # When `imp.find_module()` cannot find a package it raises ImportError. + # Here we should simulate it to keep the compatibility with older + # versions. + if not spec: + raise ImportError('No module named {!r}'.format(name)) + return spec.submodule_search_locations - def find_package_dirs(name): - return [imp.find_module(name)[1]] logger = logging.getLogger(__name__) -# -# Substitution Defaults -# - - -class DEFAULT_SUBSTITUTION(object): +class DEFAULT_SUBSTITUTION: pass -class MANDATORY_SUBSTITUTION(object): +class MANDATORY_SUBSTITUTION: pass -class NO_SUBSTITUTION(object): +class NO_SUBSTITUTION: pass -class STR_SUBSTITUTION(object): +class STR_SUBSTITUTION: pass -U_KEY_SEP = unicode('.') -U_KEY_FMT = unicode('"{0}"') +U_KEY_SEP = '.' +U_KEY_FMT = '"{0}"' -U_KEY_SEP = unicode('.') -U_KEY_FMT = unicode('"{0}"') - -class ConfigFactory(object): +class ConfigFactory: @classmethod def parse_file(cls, filename, encoding='utf-8', required=True, resolve=True, unresolved_value=DEFAULT_SUBSTITUTION): """Parse file :param filename: filename - :type filename: basestring + :type filename: str :param encoding: file encoding - :type encoding: basestring + :type encoding: str :param required: If true, raises an exception if can't load file :type required: boolean :param resolve: if true, resolve substitutions @@ -142,7 +91,7 @@ def parse_file(cls, filename, encoding='utf-8', required=True, resolve=True, unr except IOError as e: if required: raise e - logger.warn('Cannot include file %s. File does not exist or cannot be read.', filename) + logger.warning('Cannot include file %s. File does not exist or cannot be read.', filename) return [] @classmethod @@ -150,7 +99,7 @@ def parse_URL(cls, url, timeout=None, resolve=True, required=False, unresolved_v """Parse URL :param url: url to parse - :type url: basestring + :type url: str :param resolve: if true, resolve substitutions :type resolve: boolean :param unresolved_value: assigned value to unresolved substitution. @@ -164,10 +113,10 @@ def parse_URL(cls, url, timeout=None, resolve=True, required=False, unresolved_v try: with contextlib.closing(urlopen(url, timeout=socket_timeout)) as fd: - content = fd.read() if use_urllib2 else fd.read().decode('utf-8') + content = fd.read().decode('utf-8') return cls.parse_string(content, os.path.dirname(url), resolve, unresolved_value) except (HTTPError, URLError) as e: - logger.warn('Cannot include url %s. Resource is inaccessible.', url) + logger.warning('Cannot include url %s. Resource is inaccessible.', url) if required: raise e else: @@ -178,7 +127,7 @@ def parse_string(cls, content, basedir=None, resolve=True, unresolved_value=DEFA """Parse string :param content: content to parse - :type content: basestring + :type content: str :param resolve: if true, resolve substitutions :type resolve: boolean :param unresolved_value: assigned value to unresolved substitution. @@ -213,7 +162,7 @@ def create_tree(value): return create_tree(dictionary) -class ConfigParser(object): +class ConfigParser: """ Parse HOCON files: https://github.com/typesafehub/config/blob/master/HOCON.md """ @@ -235,7 +184,7 @@ def parse(cls, content, basedir=None, resolve=True, unresolved_value=DEFAULT_SUB """parse a HOCON content :param content: HOCON content to parse - :type content: basestring + :type content: str :param resolve: if true, resolve substitutions :type resolve: boolean :param unresolved_value: assigned value to unresolved substitution. @@ -245,7 +194,7 @@ def parse(cls, content, basedir=None, resolve=True, unresolved_value=DEFAULT_SUB :return: a ConfigTree or a list """ - unescape_pattern = re.compile(r'\\.') + unescape_pattern = re.compile(r'') def replace_escape_sequence(match): value = match.group(0) @@ -376,17 +325,17 @@ def _merge(a, b): @contextlib.contextmanager def set_default_white_spaces(): default = ParserElement.DEFAULT_WHITE_CHARS - ParserElement.setDefaultWhitespaceChars(' \t') + ParserElement.set_default_whitespace_chars(' \t') yield - ParserElement.setDefaultWhitespaceChars(default) + ParserElement.set_default_whitespace_chars(default) with set_default_white_spaces(): assign_expr = Forward() - true_expr = Keyword("true", caseless=True).setParseAction(replaceWith(True)) - false_expr = Keyword("false", caseless=True).setParseAction(replaceWith(False)) - null_expr = Keyword("null", caseless=True).setParseAction(replaceWith(NoneValue())) - key = QuotedString('"""', escChar='\\', unquoteResults=False) | \ - QuotedString('"', escChar='\\', unquoteResults=False) | Word(alphanums + alphas8bit + '._- /') + true_expr = Keyword("true", caseless=True).set_parse_action(replace_with(True)) + false_expr = Keyword("false", caseless=True).set_parse_action(replace_with(False)) + null_expr = Keyword("null", caseless=True).set_parse_action(replace_with(NoneValue())) + key = QuotedString('"""', esc_char='\\', unquote_results=False) | \ + QuotedString('"', esc_char='\\', unquote_results=False) | Word(alphanums + alphas8bit + '._- /') eol = Word('\n\r').suppress() eol_comma = Word('\n\r,').suppress() @@ -394,35 +343,35 @@ def set_default_white_spaces(): comment_eol = Suppress(Optional(eol_comma) + comment) comment_no_comma_eol = (comment | eol).suppress() number_expr = Regex(r'[+-]?(\d*\.\d+|\d+(\.\d+)?)([eE][+\-]?\d+)?(?=$|[ \t]*([\$\}\],#\n\r]|//))', - re.DOTALL).setParseAction(convert_number) + re.DOTALL).set_parse_action(convert_number) # multi line string using """ # Using fix described in http://pyparsing.wikispaces.com/share/view/3778969 - multiline_string = Regex('""".*?"*"""', re.DOTALL | re.UNICODE).setParseAction(parse_multi_string) + multiline_string = Regex('""".*?"*"""', re.DOTALL | re.UNICODE).set_parse_action(parse_multi_string) # single quoted line string - quoted_string = Regex(r'"(?:[^"\\\n]|\\.)*"[ \t]*', re.UNICODE).setParseAction(create_quoted_string) + quoted_string = Regex(r'"(?:[^"\\\n]|\\.)*"[ \t]*', re.UNICODE).set_parse_action(create_quoted_string) # unquoted string that takes the rest of the line until an optional comment # we support .properties multiline support which is like this: # line1 \ # line2 \ # so a backslash precedes the \n - unquoted_string = Regex(r'(?:[^^`+?!@*&"\[\{\s\]\}#,=\$\\]|\\.)+[ \t]*', re.UNICODE).setParseAction( + unquoted_string = Regex(r'(?:[^^`+?!@*&"\[\{\s\]\}#,=\$\\]|\\.)+[ \t]*', re.UNICODE).set_parse_action( unescape_string) - substitution_expr = Regex(r'[ \t]*\$\{[^\}]+\}[ \t]*').setParseAction(create_substitution) + substitution_expr = Regex(r'[ \t]*\$\{[^\}]+\}[ \t]*').set_parse_action(create_substitution) string_expr = multiline_string | quoted_string | unquoted_string value_expr = get_period_expr() | number_expr | true_expr | false_expr | null_expr | string_expr include_content = ( quoted_string | ((Keyword('url') | Keyword('file') | Keyword('package')) - Literal( - '(').suppress() - quoted_string - Literal(')').suppress()) + '(').suppress() - quoted_string - Literal(')').suppress()) ) include_expr = ( Keyword("include", caseless=True).suppress() + ( include_content | ( - Keyword("required") - Literal('(').suppress() - include_content - Literal(')').suppress() - ) - ) - ).setParseAction(include_config) + Keyword("required") - Literal('(').suppress() - include_content - Literal(')').suppress() + ) + ) + ).set_parse_action(include_config) root_dict_expr = Forward() dict_expr = Forward() @@ -442,16 +391,16 @@ def set_default_white_spaces(): # special case when we have a value assignment where the string can potentially be the remainder of the line assign_expr << Group( - key - ZeroOrMore(comment_no_comma_eol) - ( - dict_expr | (Literal('=') | Literal(':') | Literal('+=')) - ZeroOrMore( - comment_no_comma_eol) - ConcatenatedValueParser(multi_value_expr)) + key - ZeroOrMore(comment_no_comma_eol) - ( + dict_expr | (Literal('=') | Literal(':') | Literal('+=')) - ZeroOrMore( + comment_no_comma_eol) - ConcatenatedValueParser(multi_value_expr)) ) # the file can be { ... } where {} can be omitted or [] config_expr = ZeroOrMore(comment_eol | eol) + ( - list_expr | root_dict_expr | inside_root_dict_expr) + ZeroOrMore( - comment_eol | eol_comma) - config = config_expr.parseString(content, parseAll=True)[0] + list_expr | root_dict_expr | inside_root_dict_expr) + ZeroOrMore( + comment_eol | eol_comma) + config = config_expr.parse_string(content, parse_all=True)[0] if resolve: allow_unresolved = resolve and unresolved_value is not DEFAULT_SUBSTITUTION \ @@ -542,7 +491,7 @@ def _find_substitutions(cls, item): """Convert HOCON input into a JSON output :return: JSON string representation - :type return: basestring + :type return: str """ if isinstance(item, ConfigValues): return item.get_substitutions() @@ -642,11 +591,11 @@ def resolve_substitutions(cls, config, accept_unresolved=False): is_optional_resolved, resolved_value = cls._resolve_variable(config, substitution) - if isinstance(resolved_value, ConfigValues) : + if isinstance(resolved_value, ConfigValues): resolved_value = resolved_value.transform() value_to_be_substitute = resolved_value if overridden_value and not isinstance(overridden_value, ConfigValues): - value_to_be_substitute = overridden_value + value_to_be_substitute = overridden_value unresolved, _, _ = cls._do_substitute(substitution, value_to_be_substitute, is_optional_resolved) any_unresolved = unresolved or any_unresolved @@ -821,7 +770,7 @@ def postParse(self, instring, loc, token_list): if isinstance(value, list) and operator == "+=": value = ConfigValues([ConfigSubstitution(key, True, '', False, loc), value], False, loc) config_tree.put(key, value, False) - elif isinstance(value, unicode) and operator == "+=": + elif isinstance(value, str) and operator == "+=": value = ConfigValues([ConfigSubstitution(key, True, '', True, loc), ' ' + value], True, loc) config_tree.put(key, value, False) elif isinstance(value, list): diff --git a/pyhocon/config_tree.py b/src/pyhocon/config_tree.py similarity index 94% rename from pyhocon/config_tree.py rename to src/pyhocon/config_tree.py index 52ac38cc..9175df91 100644 --- a/pyhocon/config_tree.py +++ b/src/pyhocon/config_tree.py @@ -4,22 +4,16 @@ import copy from pyhocon.exceptions import ConfigException, ConfigWrongTypeException, ConfigMissingException -try: - basestring -except NameError: # pragma: no cover - basestring = str - unicode = str - -class UndefinedKey(object): +class UndefinedKey: pass -class NonExistentKey(object): +class NonExistentKey: pass -class NoneValue(object): +class NoneValue: pass @@ -137,7 +131,7 @@ def _put(self, key_path, value, append=False): else: raise ConfigWrongTypeException( - u"Cannot concatenate the list {key}: {value} to {prev_value} of {type}".format( + "Cannot concatenate the list {key}: {value} to {prev_value} of {type}".format( key='.'.join(key_path), value=value, prev_value=l_value, @@ -174,7 +168,7 @@ def _get(self, key_path, key_index=0, default=UndefinedKey): if elt is UndefinedKey: if default is UndefinedKey: raise ConfigMissingException( - u"No configuration setting found for key {key}".format(key='.'.join(key_path[:key_index + 1]))) + "No configuration setting found for key {key}".format(key='.'.join(key_path[:key_index + 1]))) else: return default @@ -190,7 +184,7 @@ def _get(self, key_path, key_index=0, default=UndefinedKey): else: if default is UndefinedKey: raise ConfigWrongTypeException( - u"{key} has type {type} rather than dict".format(key='.'.join(key_path[:key_index + 1]), + "{key} has type {type} rather than dict".format(key='.'.join(key_path[:key_index + 1]), type=type(elt).__name__)) else: return default @@ -249,7 +243,7 @@ def get_string(self, key, default=UndefinedKey): if value is None: return None - string_value = unicode(value) + string_value = str(value) if isinstance(value, bool): string_value = string_value.lower() return string_value @@ -297,7 +291,7 @@ def get_int(self, key, default=UndefinedKey): return int(value) if value is not None else None except (TypeError, ValueError): raise ConfigException( - u"{key} has type '{type}' rather than 'int'".format(key=key, type=type(value).__name__)) + "{key} has type '{type}' rather than 'int'".format(key=key, type=type(value).__name__)) def get_float(self, key, default=UndefinedKey): """Return float representation of value found at key @@ -314,7 +308,7 @@ def get_float(self, key, default=UndefinedKey): return float(value) if value is not None else None except (TypeError, ValueError): raise ConfigException( - u"{key} has type '{type}' rather than 'float'".format(key=key, type=type(value).__name__)) + "{key} has type '{type}' rather than 'float'".format(key=key, type=type(value).__name__)) def get_bool(self, key, default=UndefinedKey): """Return boolean representation of value found at key @@ -341,7 +335,7 @@ def get_bool(self, key, default=UndefinedKey): return bool_conversions[string_value] except KeyError: raise ConfigException( - u"{key} does not translate to a Boolean value".format(key=key)) + "{key} does not translate to a Boolean value".format(key=key)) def get_list(self, key, default=UndefinedKey): """Return list representation of value found at key @@ -362,13 +356,13 @@ def get_list(self, key, default=UndefinedKey): if re.match('^[1-9][0-9]*$|0', k): lst.append(v) else: - raise ConfigException(u"{key} does not translate to a list".format(key=key)) + raise ConfigException("{key} does not translate to a list".format(key=key)) return lst elif value is None: return None else: raise ConfigException( - u"{key} has type '{type}' rather than 'list'".format(key=key, type=type(value).__name__)) + "{key} has type '{type}' rather than 'list'".format(key=key, type=type(value).__name__)) def get_config(self, key, default=UndefinedKey): """Return tree config representation of value found at key @@ -387,7 +381,7 @@ def get_config(self, key, default=UndefinedKey): return None else: raise ConfigException( - u"{key} has type '{type}' rather than 'config'".format(key=key, type=type(value).__name__)) + "{key} has type '{type}' rather than 'config'".format(key=key, type=type(value).__name__)) def __getitem__(self, item): val = self.get(item) @@ -465,12 +459,12 @@ def __init__(self, iterable=[]): value.key = index -class ConfigInclude(object): +class ConfigInclude: def __init__(self, tokens): self.tokens = tokens -class ConfigValues(object): +class ConfigValues: def __init__(self, tokens, instring, loc): self.tokens = tokens self.parent = None @@ -522,7 +516,7 @@ def format_str(v, last=False): if isinstance(v, ConfigQuotedString): return v.value + ('' if last else v.ws) else: - return '' if v is None else unicode(v) + return '' if v is None else str(v) if self.has_substitution(): return self @@ -539,7 +533,8 @@ def format_str(v, last=False): tok_type = determine_type(token) if first_tok_type is not tok_type: raise ConfigWrongTypeException( - "Token '{token}' of type {tok_type} (index {index}) must be of type {req_tok_type} (line: {line}, col: {col})".format( + "Token '{token}' of type {tok_type} (index {index}) must be of type " + "{req_tok_type} (line: {line}, col: {col})".format( token=token, index=index + 1, tok_type=tok_type.__name__, @@ -601,7 +596,7 @@ def __repr__(self): # pragma: no cover return '[ConfigValues: ' + ','.join(str(o) for o in self.tokens) + ']' -class ConfigSubstitution(object): +class ConfigSubstitution: def __init__(self, variable, optional, ws, instring, loc): self.variable = variable self.optional = optional @@ -618,12 +613,12 @@ def __repr__(self): # pragma: no cover return '[ConfigSubstitution: ' + self.variable + ']' -class ConfigUnquotedString(unicode): +class ConfigUnquotedString(str): def __new__(cls, value): return super(ConfigUnquotedString, cls).__new__(cls, value) -class ConfigQuotedString(object): +class ConfigQuotedString: def __init__(self, value, ws, instring, loc): self.value = value self.ws = ws diff --git a/pyhocon/converter.py b/src/pyhocon/converter.py similarity index 96% rename from pyhocon/converter.py rename to src/pyhocon/converter.py index 6e1eed04..a47ccf24 100644 --- a/pyhocon/converter.py +++ b/src/pyhocon/converter.py @@ -10,11 +10,6 @@ from pyhocon.config_tree import NoneValue from pyhocon.period_serializer import timedelta_to_str, is_timedelta_like, timedelta_to_hocon -try: - basestring -except NameError: - basestring = str - unicode = str try: from dateutil.relativedelta import relativedelta @@ -22,13 +17,13 @@ relativedelta = None -class HOCONConverter(object): +class HOCONConverter: @classmethod def to_json(cls, config, compact=False, indent=2, level=0): """Convert HOCON input into a JSON output :return: JSON string representation - :type return: basestring + :type return: str """ lines = "" if isinstance(config, ConfigTree): @@ -62,7 +57,7 @@ def to_json(cls, config, compact=False, indent=2, level=0): lines += '\n{indent}]'.format(indent=''.rjust(level * indent, ' ')) elif is_timedelta_like(config): lines += timedelta_to_str(config) - elif isinstance(config, basestring): + elif isinstance(config, str): lines = json.dumps(config, ensure_ascii=False) elif config is None or isinstance(config, NoneValue): lines = 'null' @@ -79,7 +74,7 @@ def to_hocon(cls, config, compact=False, indent=2, level=0): """Convert HOCON input into a HOCON output :return: JSON string representation - :type return: basestring + :type return: str """ lines = "" if isinstance(config, ConfigTree): @@ -120,7 +115,7 @@ def to_hocon(cls, config, compact=False, indent=2, level=0): value=cls.to_hocon(item, compact, indent, level + 1))) lines += '\n'.join(bet_lines) lines += '\n{indent}]'.format(indent=''.rjust((level - 1) * indent, ' ')) - elif isinstance(config, basestring): + elif isinstance(config, str): if '\n' in config and len(config) > 1: lines = '"""{value}"""'.format(value=config) # multilines else: @@ -154,7 +149,7 @@ def to_yaml(cls, config, compact=False, indent=2, level=0): """Convert HOCON input into a YAML output :return: YAML string representation - :type return: basestring + :type return: str """ lines = "" if isinstance(config, ConfigTree): @@ -182,7 +177,7 @@ def to_yaml(cls, config, compact=False, indent=2, level=0): lines += '\n'.join(bet_lines) elif is_timedelta_like(config): lines += timedelta_to_str(config) - elif isinstance(config, basestring): + elif isinstance(config, str): # if it contains a \n then it's multiline lines = config.split('\n') if len(lines) == 1: @@ -204,7 +199,7 @@ def to_properties(cls, config, compact=False, indent=2, key_stack=None): """Convert HOCON input into a .properties output :return: .properties string representation - :type return: basestring + :type return: str :return: """ key_stack = key_stack or [] @@ -224,7 +219,7 @@ def escape_value(value): lines.append(cls.to_properties(item, compact, indent, stripped_key_stack + [str(index)])) elif is_timedelta_like(config): lines.append('.'.join(stripped_key_stack) + ' = ' + timedelta_to_str(config)) - elif isinstance(config, basestring): + elif isinstance(config, str): lines.append('.'.join(stripped_key_stack) + ' = ' + escape_value(config)) elif config is True: lines.append('.'.join(stripped_key_stack) + ' = true') @@ -290,4 +285,3 @@ def _escape_match(cls, match): @classmethod def _escape_string(cls, string): return re.sub(r'[\x00-\x1F"\\]', cls._escape_match, string) - diff --git a/pyhocon/exceptions.py b/src/pyhocon/exceptions.py similarity index 100% rename from pyhocon/exceptions.py rename to src/pyhocon/exceptions.py diff --git a/pyhocon/period_parser.py b/src/pyhocon/period_parser.py similarity index 95% rename from pyhocon/period_parser.py rename to src/pyhocon/period_parser.py index efa7a480..fc25f1a2 100644 --- a/pyhocon/period_parser.py +++ b/src/pyhocon/period_parser.py @@ -64,8 +64,8 @@ def get_period_expr(): return Combine( Word(nums)('value') + ZeroOrMore(Literal(" ")).suppress() + Or(period_types)('unit') + WordEnd( alphanums).suppress() - ).setParseAction(convert_period) + ).set_parse_action(convert_period) def parse_period(content): - return get_period_expr().parseString(content, parseAll=True)[0] + return get_period_expr().parse_string(content, parse_all=True)[0] diff --git a/pyhocon/period_serializer.py b/src/pyhocon/period_serializer.py similarity index 100% rename from pyhocon/period_serializer.py rename to src/pyhocon/period_serializer.py diff --git a/pyhocon/tool.py b/src/pyhocon/tool.py similarity index 100% rename from pyhocon/tool.py rename to src/pyhocon/tool.py diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index e81fdcd1..50117f5a 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -7,27 +7,23 @@ from collections import OrderedDict from datetime import timedelta -try: - # Python 3 - from urllib.request import pathname2url -except ImportError: - # Python 2 - from urllib import pathname2url +from urllib.request import pathname2url -import mock +from unittest import mock import pytest from pyparsing import ParseBaseException, ParseException, ParseSyntaxException from pyhocon import (ConfigFactory, ConfigParser, ConfigSubstitutionException, - ConfigTree, HOCONConverter) + ConfigTree, HOCONConverter) from pyhocon.exceptions import (ConfigException, ConfigMissingException, - ConfigWrongTypeException) + ConfigWrongTypeException) try: from dateutil.relativedelta import relativedelta as period except Exception: from datetime import timedelta as period + class TestConfigParser(object): def test_parse_simple_value(self): config = ConfigFactory.parse_string( @@ -118,7 +114,6 @@ def test_parse_with_enclosing_brace_and_period_like_value(self): assert config.get_string('a.b') == '5' assert config.get_string('a.y_min') == '42' - def test_issue_324(self): config = ConfigFactory.parse_string("a { c = 3\nd = 4 }") assert config["a"]["c"] == 3 @@ -199,7 +194,6 @@ def test_parse_with_list_mixed_types_with_durations_and_trailing_comma(self): config['b'] == ['a', 1, '10 weeks', '5 minutes'] ) - def test_parse_with_enclosing_square_bracket(self): config = ConfigFactory.parse_string("[1, 2, 3]") assert config == [1, 2, 3] @@ -1757,12 +1751,11 @@ def test_override_optional_substitution(self): result = ${test} """) assert config == { - 'a' : 3, + 'a': 3, 'test': 3, 'result': 3 } - def test_substitution_cycle(self): with pytest.raises(ConfigSubstitutionException): ConfigFactory.parse_string( @@ -2701,7 +2694,6 @@ def test_triple_quotes_keys_triple_quotes_values_second_separator(self): try: from dateutil.relativedelta import relativedelta - @pytest.mark.parametrize('data_set', [ ('a: 1 months', relativedelta(months=1)), ('a: 1months', relativedelta(months=1)), diff --git a/tests/test_config_tree.py b/tests/test_config_tree.py index 93fb20cd..2b293f8a 100644 --- a/tests/test_config_tree.py +++ b/tests/test_config_tree.py @@ -4,7 +4,7 @@ from pyhocon.exceptions import ( ConfigMissingException, ConfigWrongTypeException, ConfigException) from pyhocon.config_parser import ConfigFactory -from pyhocon.tool import HOCONConverter +from pyhocon import HOCONConverter class TestConfigTree(object): diff --git a/tests/test_periods.py b/tests/test_periods.py index f5eb8e60..82c2e588 100644 --- a/tests/test_periods.py +++ b/tests/test_periods.py @@ -62,7 +62,6 @@ def test_parse_string_with_duration(data_set): try: from dateutil.relativedelta import relativedelta - @pytest.mark.parametrize('data_set', [ ('1 months', relativedelta(months=1)), ('1months', relativedelta(months=1)), @@ -82,7 +81,6 @@ def test_parse_string_with_duration_optional_units(data_set): assert parsed == data_set[1] - def test_format_relativedelta(): for time_delta, expected_result in ((relativedelta(seconds=0), '0 seconds'), @@ -93,6 +91,7 @@ def test_format_relativedelta(): (relativedelta(minutes=43), '43 minutes'),): assert expected_result == timedelta_to_hocon(time_delta) except ImportError: + relativedelta = None pass diff --git a/tox.ini b/tox.ini index 531e4c15..e669a926 100644 --- a/tox.ini +++ b/tox.ini @@ -1,20 +1,21 @@ [tox] -envlist = flake8, py{27,38,39,310,311,312} +envlist = flake8, py{310,311,312,313,314} [testenv] -passenv = TRAVIS TRAVIS_JOB_ID TRAVIS_BRANCH +passenv = + TRAVIS + TRAVIS_JOB_ID + TRAVIS_BRANCH deps = - pytest + pytest-cov coveralls python-dateutil>=2.8.0 - # for python 3.4 - typing commands = - coverage run --source=pyhocon setup.py test + pytest --cov=pyhocon --cov-report=term-missing {posargs} coverage report -m coveralls [testenv:flake8] basepython = python deps = flake8 -commands = flake8 pyhocon tests setup.py +commands = flake8 --extend-ignore=E1,W1 --max-line-length=128 --exclude=.tox,site-packages,dist,build,*.egg-info,*.egg,docs