From 8270825ab541d913bd30544d3b310bcf58bad9cb Mon Sep 17 00:00:00 2001 From: david-i-berry Date: Fri, 14 Apr 2023 19:11:03 +0200 Subject: [PATCH 1/5] Update .gitignore Addition of /vocab route to flask app. New postgresql non spatial provider. Update to plugin name Update to collection to add in license information if present. Update to collection to add in license information if present. Update to collection to add in license information if present. Update to collection to add in license information if present. Update to collection to add in license information if present. Update to collection to add in license information if present. Update to collection to add in license information if present. Update to collection to add in license information if present. Update to collection to add in license information if present. CRUD operations for psql. CRUD operations for psql. CRUD operations for psql. CRUD operations for psql. shapely 2 shapely 2 debugging debugging debugging debugging debugging debugging debugging sqlalchemy < 2 ! testing testing addition of delete addition of delete --- .gitignore | 5 + pygeoapi/api.py | 650 ++++++++++++++++++ pygeoapi/flask_app.py | 44 ++ pygeoapi/formatter/csvTable.py | 93 +++ pygeoapi/plugin.py | 8 +- pygeoapi/provider/postgresql.py | 90 ++- pygeoapi/provider/postgresql_nonspatial.py | 365 ++++++++++ .../templates/collections/collection.html | 15 + pygeoapi/templates/landing_page.html | 6 + pygeoapi/templates/vocabularies/index.html | 35 + .../templates/vocabularies/items/index.html | 91 +++ .../templates/vocabularies/items/item.html | 93 +++ .../templates/vocabularies/vocabulary.html | 65 ++ requirements.txt | 2 +- 14 files changed, 1557 insertions(+), 5 deletions(-) create mode 100644 pygeoapi/formatter/csvTable.py create mode 100644 pygeoapi/provider/postgresql_nonspatial.py create mode 100644 pygeoapi/templates/vocabularies/index.html create mode 100644 pygeoapi/templates/vocabularies/items/index.html create mode 100644 pygeoapi/templates/vocabularies/items/item.html create mode 100644 pygeoapi/templates/vocabularies/vocabulary.html diff --git a/.gitignore b/.gitignore index 97c5b8970..f1e4422fd 100644 --- a/.gitignore +++ b/.gitignore @@ -121,3 +121,8 @@ examples/django/sample_project/db.sqlite3 # Pre-commit hooks config file .pre-commit-config.yaml +.idea/inspectionProfiles/profiles_settings.xml +.idea/inspectionProfiles/Project_Default.xml + +# Pcycharm IDE +.idea/ diff --git a/pygeoapi/api.py b/pygeoapi/api.py index 82c73a341..330eb66d0 100644 --- a/pygeoapi/api.py +++ b/pygeoapi/api.py @@ -4146,6 +4146,656 @@ def _set_content_crs_header( headers['Content-Crs'] = f'<{content_crs_uri}>' + def get_vocabularies_url(self): + return f"{self.base_url}/vocabularies" + + @gzip + @pre_process + @jsonldify + def describe_vocabularies(self, request: Union[APIRequest, Any], + vocab=None) -> Tuple[dict, int, str]: + """ + Provide vocabulary metadata + + :param request: A request object + :param vocab: vocab identifier, defaults to None to obtain + information about all vocabularies + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + headers = request.get_response_headers() + + fcm = { + 'vocabularies': [], + 'links': [] + } + + vocabularies = filter_dict_by_key_value(self.config['resources'], + 'type', 'vocabulary') + + if all([vocab is not None, vocab not in vocabularies.keys()]): + msg = 'Vocabulary not found' + return self.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + if vocab is not None: + vocabularies_dict = { + key: value for key, value in vocabularies.items() if + key == vocab + # noqa + } + else: + vocabularies_dict = vocabularies + + LOGGER.debug("Creating vocabularies") + + for key, value in vocabularies_dict.items(): + vocab_data = get_provider_default(value['providers']) + vocab_data_type = vocab_data['type'] + vocab_data_format = None + + if 'format' in vocab_data: + vocab_data_format = vocab_data['format'] + + LOGGER.debug(value) + + vocab_ = { + 'id': key, + 'title': l10n.translate(value['title'], request.locale), + 'description': l10n.translate(value['description'], + request.locale), # noqa + 'links': [] + } + + LOGGER.debug('Processing configured vocabulary links') + for link in l10n.translate(value['links'], request.locale): + lnk = { + 'type': link['type'], + 'rel': link['rel'], + 'title': l10n.translate(link['title'], request.locale), + 'href': l10n.translate(link['href'], request.locale), + } + if 'hreflang' in link: + lnk['hreflang'] = l10n.translate( + link['hreflang'], request.locale) + content_length = link.get('length', 0) + if content_length > 0: + lnk['length'] = content_length + + vocab_['links'].append(lnk) + + # TODO: provide translations + LOGGER.debug('Adding JSON and HTML link relations') + vocab_['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'root', + 'title': 'The landing page of this server as JSON', + 'href': f"{self.base_url}?f={F_JSON}" + }) + vocab_['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'root', + 'title': 'The landing page of this server as HTML', + 'href': f"{self.base_url}?f={F_HTML}" + }) + vocab_['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'title': 'This document as JSON', + 'href': f'{self.get_vocabularies_url()}/{key}?f={F_JSON}' + }) + vocab_['links'].append({ + 'type': FORMAT_TYPES[F_JSONLD], + 'rel': request.get_linkrel(F_JSONLD), + 'title': 'This document as RDF (JSON-LD)', + 'href': f'{self.get_vocabularies_url()}/{key}?f={F_JSONLD}' + }) + vocab_['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': 'This document as HTML', + 'href': f'{self.get_vocabularies_url()}/{key}?f={F_HTML}' + }) + + if vocab is not None and key == vocab: + fcm = vocab_ + break + + fcm['vocabularies'].append(vocab_) + + if vocab is None: + vocabulary_url = self.get_vocabularies_url() + response = { + 'vocabularies': vocabularies, + 'links': [{ + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'title': 'This document as JSON', + 'href': f'{vocabulary_url}?f={F_JSON}' + }, { + 'type': FORMAT_TYPES[F_JSONLD], + 'rel': request.get_linkrel(F_JSONLD), + 'title': 'This document as RDF (JSON-LD)', + 'href': f'{vocabulary_url}?f={F_JSONLD}' + }, { + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': 'This document as HTML', + 'href': f'{vocabulary_url}?f={F_HTML}' + }] + } + + if request.format == F_HTML: # render + fcm['vocabularies_path'] = self.get_vocabularies_url() + if vocab is not None: + response = render_j2_template(self.tpl_config, + 'vocabularies/vocabulary.html', + fcm, request.locale) + else: + response = render_j2_template(self.tpl_config, + 'vocabularies/index.html', fcm, + request.locale) + + return headers, HTTPStatus.OK, response + + # ToDo - add F_JSONLD format + + return headers, HTTPStatus.OK, to_json(fcm, self.pretty_print) + + @gzip + @pre_process + def get_vocabulary_items( + self, request: Union[APIRequest, Any], + vocab) -> Tuple[dict, int, str]: + """ + Queries vocabulary + + :param request: A request object + :param vocab: vocabulary name + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(PLUGINS['formatter'].keys()): + return self.get_format_exception(request) + + # Set Content-Language to system locale until provider locale + # has been determined + headers = request.get_response_headers(SYSTEM_LOCALE, + **self.api_headers) + properties = [] + reserved_fieldnames = ['bbox', 'bbox-crs', 'crs', 'f', 'lang', 'limit', + 'offset', 'resulttype', 'datetime', 'sortby', + 'properties', 'skipGeometry', 'q', + 'filter', 'filter-lang'] + + vocabularies = filter_dict_by_key_value(self.config['resources'], + 'type', 'vocabulary') + + if vocab not in vocabularies.keys(): + msg = 'vocabulary not found' + return self.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Processing query parameters') + + LOGGER.debug('Processing offset parameter') + try: + offset = int(request.params.get('offset')) + if offset < 0: + msg = 'offset value should be positive or zero' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except TypeError as err: + LOGGER.warning(err) + offset = 0 + except ValueError: + msg = 'offset value should be an integer' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing limit parameter') + try: + limit = int(request.params.get('limit')) + # TODO: We should do more validation, against the min and max + # allowed by the server configuration + if limit <= 0: + msg = 'limit value should be strictly positive' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except TypeError as err: + LOGGER.warning(err) + limit = int(self.config['server']['limit']) + except ValueError: + msg = 'limit value should be an integer' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + resulttype = request.params.get('resulttype') or 'results' + + datetime_ = '' + if 'extents' in vocabularies[vocab]: + LOGGER.debug('Processing datetime parameter') + datetime_ = request.params.get('datetime') + try: + datetime_ = validate_datetime(vocabularies[vocab]['extents'], + datetime_) + except ValueError as err: + msg = str(err) + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('processing q parameter') + q = request.params.get('q') or None + + LOGGER.debug('Loading provider') + + try: + provider_def = get_provider_by_type( + vocabularies[vocab]['providers'], 'feature') + p = load_plugin('provider', provider_def) + except ProviderTypeError: + try: + provider_def = get_provider_by_type( + vocabularies[vocab]['providers'], 'record') + p = load_plugin('provider', provider_def) + except ProviderTypeError: + msg = 'Invalid provider type' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', msg) + except ProviderConnectionError: + msg = 'connection error (check logs)' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + except ProviderQueryError: + msg = 'query error (check logs)' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + + LOGGER.debug('processing property parameters') + for k, v in request.params.items(): + if k not in reserved_fieldnames and k in list(p.fields.keys()): + LOGGER.debug(f'Adding property filter {k}={v}') + properties.append((k, v)) + + LOGGER.debug('processing sort parameter') + val = request.params.get('sortby') + + if val is not None: + sortby = [] + sorts = val.split(',') + for s in sorts: + prop = s + order = '+' + if s[0] in ['+', '-']: + order = s[0] + prop = s[1:] + + if prop not in p.fields.keys(): + msg = 'bad sort property' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + sortby.append({'property': prop, 'order': order}) + else: + sortby = [] + + LOGGER.debug('processing properties parameter') + val = request.params.get('properties') + + if val is not None: + select_properties = val.split(',') + properties_to_check = set(p.properties) | set(p.fields.keys()) + + if (len(list(set(select_properties) - + set(properties_to_check))) > 0): + msg = 'unknown properties specified' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + else: + select_properties = [] + + LOGGER.debug('processing filter parameter') + filter_ = None + + # Get provider locale (if any) + prv_locale = l10n.get_plugin_locale(provider_def, request.raw_locale) + + LOGGER.debug('Querying provider') + LOGGER.debug(f'offset: {offset}') + LOGGER.debug(f'limit: {limit}') + LOGGER.debug(f'resulttype: {resulttype}') + LOGGER.debug(f'sortby: {sortby}') + LOGGER.debug(f'datetime: {datetime_}') + LOGGER.debug(f'properties: {properties}') + LOGGER.debug(f'select properties: {select_properties}') + LOGGER.debug(f'language: {prv_locale}') + LOGGER.debug(f'q: {q}') + + try: + content = p.query(offset=offset, limit=limit, + resulttype=resulttype, properties=properties, + datetime_=datetime_, sortby=sortby, + select_properties=select_properties, + q=q, language=prv_locale) + LOGGER.debug(content) + # content is now a feature collection + except ProviderConnectionError as err: + LOGGER.error(err) + msg = 'connection error (check logs)' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + except ProviderQueryError as err: + LOGGER.error(err) + msg = 'query error (check logs)' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + except ProviderGenericError as err: + LOGGER.error(err) + msg = 'generic error (check logs)' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + + serialized_query_params = '' + for k, v in request.params.items(): + if k not in ('f', 'offset'): + serialized_query_params += '&' + serialized_query_params += urllib.parse.quote(k, safe='') + serialized_query_params += '=' + serialized_query_params += urllib.parse.quote(str(v), safe=',') + + # TODO: translate titles + uri = f'{self.get_vocabularies_url()}/{vocab}/items' + content['links'] = [{ + 'type': 'application/json', + 'rel': request.get_linkrel(F_JSON), + 'title': 'This document as JSON', + 'href': f'{uri}?f={F_JSON}{serialized_query_params}' + }, { + 'rel': request.get_linkrel(F_JSONLD), + 'type': FORMAT_TYPES[F_JSONLD], + 'title': 'This document as RDF (JSON-LD)', + 'href': f'{uri}?f={F_JSONLD}{serialized_query_params}' + }, { + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': 'This document as HTML', + 'href': f'{uri}?f={F_HTML}{serialized_query_params}' + }] + + if offset > 0: + prev = max(0, offset - limit) + content['links'].append( + { + 'type': 'application/json', + 'rel': 'prev', + 'title': 'items (prev)', + 'href': f'{uri}?offset={prev}{serialized_query_params}' + }) + + if len(content['items']) == limit: + next_ = offset + limit + content['links'].append( + { + 'type': 'application/json', + 'rel': 'next', + 'title': 'items (next)', + 'href': f'{uri}?offset={next_}{serialized_query_params}' + }) + + content['links'].append( + { + 'type': FORMAT_TYPES[F_JSON], + 'title': l10n.translate( + vocabularies[vocab]['title'], request.locale), + 'rel': 'vocabulary', + 'href': uri + }) + + # Set response language to requested provider locale + # (if it supports language) and/or otherwise the requested pygeoapi + # locale (or fallback default locale) + l10n.set_response_language(headers, prv_locale, request.locale) + + if request.format == F_HTML: # render + # For constructing proper URIs to items + + content['items_path'] = uri + content['vocab_path'] = '/'.join(uri.split('/')[:-1]) + content['vocabularies_path'] = self.get_vocabularies_url() + + content['offset'] = offset + + content['id_field'] = p.id_field + if p.uri_field is not None: + content['uri_field'] = p.uri_field + if p.title_field is not None: + content['title_field'] = l10n.translate(p.title_field, + request.locale) + # If title exists, use it as id in html templates + content['id_field'] = content['title_field'] + content = render_j2_template(self.tpl_config, + 'vocabularies/items/index.html', + content, request.locale) + return headers, HTTPStatus.OK, content + elif request.format == 'csv': # render + formatter = load_plugin('formatter', + {'name': 'CSVTable'}) + + try: + content = formatter.write( + data=content, + options={ + 'provider_def': get_provider_by_type( + vocabularies[vocab]['providers'], + 'feature') + } + ) + except FormatterSerializationError as err: + LOGGER.error(err) + msg = 'Error serializing output' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + + headers['Content-Type'] = formatter.mimetype + + if p.filename is None: + filename = f'{vocab}.csv' + else: + filename = f'{p.filename}' + + cd = f'attachment; filename="{filename}"' + headers['Content-Disposition'] = cd + + return headers, HTTPStatus.OK, content + + elif request.format == F_JSONLD: + msg = "JSON LD not implemented for vocabularies" + raise NotImplementedError(msg) + + return headers, HTTPStatus.OK, to_json(content, self.pretty_print) + + @gzip + @pre_process + def get_vocabulary_item(self, request: Union[APIRequest, Any], + vocab, identifier) -> Tuple[dict, int, str]: + """ + Get a single vocabulary item + + :param request: A request object + :param vocab: vocab name + :param identifier: item identifier + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + + # Set Content-Language to system locale until provider locale + # has been determined + headers = request.get_response_headers(SYSTEM_LOCALE, + **self.api_headers) + LOGGER.debug('Processing query parameters') + + vocabularies = filter_dict_by_key_value(self.config['resources'], + 'type', 'vocabulary') + + if vocab not in vocabularies.keys(): + msg = 'vocabulary not found' + return self.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Loading provider') + + try: + provider_def = get_provider_by_type( + vocabularies[vocab]['providers'], 'feature') + p = load_plugin('provider', provider_def) + except ProviderTypeError: + try: + provider_def = get_provider_by_type( + vocabularies[vocab]['providers'], 'record') + p = load_plugin('provider', provider_def) + except ProviderTypeError: + msg = 'Invalid provider type' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + # Get provider language (if any) + prv_locale = l10n.get_plugin_locale(provider_def, request.raw_locale) + + try: + LOGGER.debug(f'Fetching id {identifier}') + content = p.get(identifier, language=prv_locale) + except ProviderConnectionError as err: + LOGGER.error(err) + msg = 'connection error (check logs)' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + except ProviderItemNotFoundError: + msg = 'identifier not found' + return self.get_exception(HTTPStatus.NOT_FOUND, headers, + request.format, 'NotFound', msg) + except ProviderQueryError as err: + LOGGER.error(err) + msg = 'query error (check logs)' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + except ProviderGenericError as err: + LOGGER.error(err) + msg = 'generic error (check logs)' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + + if content is None: + msg = 'identifier not found' + return self.get_exception(HTTPStatus.BAD_REQUEST, headers, + request.format, 'NotFound', msg) + + uri = content['properties'].get(p.uri_field) if p.uri_field else \ + f'{self.get_vocabularies_url()}/{vocab}/items/{identifier}' + + if 'links' not in content: + content['links'] = [] + if content['links'] is None: + content['links'] = [] + + content['links'].extend([{ + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'root', + 'title': 'The landing page of this server as JSON', + 'href': f"{self.base_url}?f={F_JSON}" + }, { + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'root', + 'title': 'The landing page of this server as HTML', + 'href': f"{self.base_url}?f={F_HTML}" + }, { + 'rel': request.get_linkrel(F_JSON), + 'type': 'application/json', + 'title': 'This document as JSON', + 'href': f'{uri}?f={F_JSON}' + }, { + 'rel': request.get_linkrel(F_JSONLD), + 'type': FORMAT_TYPES[F_JSONLD], + 'title': 'This document as RDF (JSON-LD)', + 'href': f'{uri}?f={F_JSONLD}' + }, { + 'rel': request.get_linkrel(F_HTML), + 'type': FORMAT_TYPES[F_HTML], + 'title': 'This document as HTML', + 'href': f'{uri}?f={F_HTML}' + }, { + 'rel': 'vocabulary', + 'type': FORMAT_TYPES[F_JSON], + 'title': l10n.translate(vocabularies[vocab]['title'], + request.locale), + 'href': f'{self.get_vocabularies_url()}/{vocab}' + }]) + + if 'prev' in content: + content['links'].append({ + 'rel': 'prev', + 'type': FORMAT_TYPES[request.format], + 'href': f"{self.get_vocabularies_url()}/{vocab}/items/{content['prev']}?f={request.format}" + # noqa + }) + if 'next' in content: + content['links'].append({ + 'rel': 'next', + 'type': FORMAT_TYPES[request.format], + 'href': f"{self.get_vocabularies_url()}/{vocab}/items/{content['next']}?f={request.format}" + # noqa + }) + + # Set response language to requested provider locale + # (if it supports language) and/or otherwise the requested pygeoapi + # locale (or fallback default locale) + l10n.set_response_language(headers, prv_locale, request.locale) + + if request.format == F_HTML: # render + content['title'] = l10n.translate(vocabularies[vocab]['title'], + request.locale) + content['id_field'] = p.id_field + if p.uri_field is not None: + content['uri_field'] = p.uri_field + if p.title_field is not None: + content['title_field'] = l10n.translate(p.title_field, + request.locale) + content['vocabularies_path'] = self.get_vocabularies_url() + + content = render_j2_template(self.tpl_config, + 'vocabularies/items/item.html', + content, request.locale) + return headers, HTTPStatus.OK, content + + elif request.format == F_JSONLD: + msg = "JSONLD not yet implemented for vocab" + raise NotImplementedError(msg) + + return headers, HTTPStatus.OK, to_json(content, self.pretty_print) def validate_bbox(value=None) -> list: """ diff --git a/pygeoapi/flask_app.py b/pygeoapi/flask_app.py index 74be94dd1..94f929969 100644 --- a/pygeoapi/flask_app.py +++ b/pygeoapi/flask_app.py @@ -354,6 +354,50 @@ def get_processes(process_id=None): return get_response(api_.describe_processes(request, process_id)) +@BLUEPRINT.route('/vocabularies') +@BLUEPRINT.route('/vocabularies/') +def get_vocab(vocab_id=None): + """ + OGC API - Processes description endpoint + + :param process_id: process identifier + + :returns: HTTP response + """ + if vocab_id is None: + return get_response(api_.describe_vocabularies(request, vocab_id)) + else: + if request.method == 'GET': # list items + return get_response( + api_.get_vocabulary_items(request, vocab_id)) + + + +@BLUEPRINT.route('/vocabularies//items', + methods=['GET', 'POST', 'OPTIONS'], + provide_automatic_options=False) +@BLUEPRINT.route('/vocabularies//items/', + methods=['GET'], + provide_automatic_options=False) +def vocab_items(vocab_id, item_id=None): + """ + OGC API collections items endpoint + + :param collection_id: collection identifier + :param item_id: item identifier + + :returns: HTTP response + """ + + if item_id is None: + if request.method == 'GET': # list items + return get_response( + api_.get_vocabulary_items(request, vocab_id)) + else: + return get_response( + api_.get_vocabulary_item(request, vocab_id, item_id)) + + @BLUEPRINT.route('/jobs') @BLUEPRINT.route('/jobs/', methods=['GET', 'DELETE']) diff --git a/pygeoapi/formatter/csvTable.py b/pygeoapi/formatter/csvTable.py new file mode 100644 index 000000000..45a3f068d --- /dev/null +++ b/pygeoapi/formatter/csvTable.py @@ -0,0 +1,93 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# +# Copyright (c) 2022 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import io +import logging + +import unicodecsv as csv + +from pygeoapi.formatter.base import BaseFormatter, FormatterSerializationError + +LOGGER = logging.getLogger(__name__) + + +class CSVFormatter(BaseFormatter): + """CSV formatter""" + + def __init__(self, formatter_def: dict): + """ + Initialize object + + :param formatter_def: formatter definition + + :returns: `pygeoapi.formatter.csv_.CSVFormatter` + """ + + geom = False + if 'geom' in formatter_def: + geom = formatter_def['geom'] + + super().__init__({'name': 'csv', 'geom': geom}) + self.mimetype = 'text/csv; charset=utf-8' + + def write(self, options: dict = {}, data: dict = None) -> str: + """ + Generate data in CSV format + + :param options: CSV formatting options + :param data: dict of GeoJSON data + + :returns: string representation of format + """ + + is_point = False + try: + fields = list(data['items'][0].keys()) + except IndexError: + LOGGER.error('no features') + return str() + + LOGGER.debug(f'CSV fields: {fields}') + + try: + output = io.BytesIO() + writer = csv.DictWriter(output, fields) + writer.writeheader() + + for item in data['items']: + writer.writerow(item) + + except ValueError as err: + LOGGER.error(err) + raise FormatterSerializationError('Error writing CSV output') + + return output.getvalue() + + def __repr__(self): + return f' {self.name}' diff --git a/pygeoapi/plugin.py b/pygeoapi/plugin.py index a2d63c25e..bb991856d 100644 --- a/pygeoapi/plugin.py +++ b/pygeoapi/plugin.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2022 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -58,10 +58,12 @@ 'TinyDBCatalogue': 'pygeoapi.provider.tinydb_.TinyDBCatalogueProvider', 'WMSFacade': 'pygeoapi.provider.wms_facade.WMSFacadeProvider', 'xarray': 'pygeoapi.provider.xarray_.XarrayProvider', - 'xarray-edr': 'pygeoapi.provider.xarray_edr.XarrayEDRProvider' + 'xarray-edr': 'pygeoapi.provider.xarray_edr.XarrayEDRProvider', + 'PostgreSQL-nonspatial': 'pygeoapi.provider.postgresql_nonspatial.PostgreSQLNSProvider' # noqa }, 'formatter': { - 'CSV': 'pygeoapi.formatter.csv_.CSVFormatter' + 'CSV': 'pygeoapi.formatter.csv_.CSVFormatter', + 'CSVTable': 'pygeoapi.formatter.csvTable.CSVFormatter' }, 'process': { 'HelloWorld': 'pygeoapi.process.hello_world.HelloWorldProcessor' diff --git a/pygeoapi/provider/postgresql.py b/pygeoapi/provider/postgresql.py index e44729c76..e861812e0 100644 --- a/pygeoapi/provider/postgresql.py +++ b/pygeoapi/provider/postgresql.py @@ -49,15 +49,18 @@ # psql -U postgres -h 127.0.0.1 -p 5432 test import logging +import uuid from copy import deepcopy from geoalchemy2 import Geometry # noqa - this isn't used explicitly but is needed to process Geometry columns from geoalchemy2.functions import ST_MakeEnvelope from geoalchemy2.shape import to_shape +import json from pygeofilter.backends.sqlalchemy.evaluate import to_filter import pyproj import shapely -from sqlalchemy import create_engine, MetaData, PrimaryKeyConstraint, asc, desc +from sqlalchemy import (create_engine, MetaData, PrimaryKeyConstraint, asc, + desc, insert, update, delete) from sqlalchemy.engine import URL from sqlalchemy.exc import InvalidRequestError, OperationalError from sqlalchemy.ext.automap import automap_base @@ -243,6 +246,78 @@ def get(self, identifier, crs_transform_spec=None, **kwargs): return feature + def create(self, item): + """ + Create a new item (from geojson) + + :param item: geojson string with with item to add + + :returns: identifier of created item + """ + # first convert json string to dict + feature = json.loads(item) + # check we have an ID, if not create a random one + if feature['id'] in (None, ''): + feature['id'] = str(uuid.uuid4()) + # convert from dict to object to insert into table + geom = self._feature_to_sqlalchemy(feature) + # now insert + with Session(self._engine) as session: + statement = (insert(self.table_model).values(geom)) + session.execute(statement) + session.commit() + + return feature['id'] + + def update(self, identifier, item): + """ + Updates an existing item + + :param identifier: feature id + :param item: `dict` of partial or full item + + :returns: `bool` of update result + """ + LOGGER.debug(f"Updating item {identifier}") + # first convert json string to dict + feature = json.loads(item) + # convert from dict to object to insert into table + geom = self._feature_to_sqlalchemy(feature) + # get id column + id_column = getattr(self.table_model, self.id_field) + # now insert + with Session(self._engine) as session: + statement = ( + update(self.table_model). + where(id_column == identifier). + values(geom)) + session.execute(statement) + session.commit() + + return True + + def delete(self, identifier): + """ + Deletes an existing item + + :param identifier: item id + + :returns: `bool` of deletion result + """ + + LOGGER.debug(f'Deleting item {identifier}') + + id_column = getattr(self.table_model, self.id_field) + with Session(self._engine) as session: + statement = ( + delete(self.table_model). + where(id_column == identifier) + ) + session.execute(statement) + session.commit() + + return True + def _store_db_parameters(self, parameters): self.db_user = parameters.get('user') self.db_host = parameters.get('host') @@ -375,6 +450,19 @@ def _sqlalchemy_to_feature(self, item, crs_transform_out=None): return feature + def _feature_to_sqlalchemy(self, feature): + # get geometry and convert to WKT + result = {} + result[self.id_field] = feature['id'] + geom = shapely.to_wkt( + shapely.from_geojson(json.dumps( feature['geometry'])) + ) + result[self.geom] = f"SRID=4326;{geom}" + for column in self.table_model.__table__.columns: + if column.name not in (self.id_field, self.geom): + result[column.name] = feature['properties'][column.name] + return result + def _get_order_by_clauses(self, sort_by, table_model): # Build sort_by clauses if provided clauses = [] diff --git a/pygeoapi/provider/postgresql_nonspatial.py b/pygeoapi/provider/postgresql_nonspatial.py new file mode 100644 index 000000000..d2c169d72 --- /dev/null +++ b/pygeoapi/provider/postgresql_nonspatial.py @@ -0,0 +1,365 @@ +# ================================================================= +# +# Authors: Jorge Samuel Mendes de Jesus +# Tom Kralidis +# Mary Bucknell +# John A Stevenson +# Colin Blackburn +# Francesco Bartoli +# +# Copyright (c) 2018 Jorge Samuel Mendes de Jesus +# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Francesco Bartoli +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +# Testing local docker: +# docker run --name "postgis" \ +# -v postgres_data:/var/lib/postgresql -p 5432:5432 \ +# -e ALLOW_IP_RANGE=0.0.0.0/0 \ +# -e POSTGRES_USER=postgres \ +# -e POSTGRES_PASS=postgres \ +# -e POSTGRES_DBNAME=test \ +# -d -t kartoza/postgis + +# Import dump: +# gunzip < tests/data/hotosm_bdi_waterways.sql.gz | +# psql -U postgres -h 127.0.0.1 -p 5432 test + +import logging + +from copy import deepcopy +from sqlalchemy import create_engine, MetaData, PrimaryKeyConstraint, asc, desc +from sqlalchemy.exc import InvalidRequestError, OperationalError +from sqlalchemy.ext.automap import automap_base +from sqlalchemy.orm import Session, load_only +from sqlalchemy.sql.expression import and_ + +from pygeoapi.provider.base import BaseProvider, \ + ProviderConnectionError, ProviderQueryError, ProviderItemNotFoundError + + +_ENGINE_STORE = {} +_TABLE_MODEL_STORE = {} +LOGGER = logging.getLogger(__name__) + + +class PostgreSQLNSProvider(BaseProvider): + """Generic provider for Postgresql based on psycopg2 + using sync approach and server side + cursor (using support class DatabaseCursor) + """ + def __init__(self, provider_def): + """ + PostgreSQLProvider Class constructor + + :param provider_def: provider definitions from yml pygeoapi-config. + data,id_field, name set in parent class + data contains the connection information + for class DatabaseCursor + + :returns: pygeoapi.provider.base.PostgreSQLProvider + """ + LOGGER.debug('Initialising PostgreSQL provider.') + super().__init__(provider_def) + + self.table = provider_def['table'] + self.id_field = provider_def['id_field'] + + LOGGER.debug(f'Name: {self.name}') + LOGGER.debug(f'Table: {self.table}') + LOGGER.debug(f'ID field: {self.id_field}') + + # Read table information from database + self._store_db_parameters(provider_def['data']) + self._engine, self.table_model = self._get_engine_and_table_model() + LOGGER.debug(f'DB connection: {repr(self._engine.url)}') + self.fields = self.get_fields() + + def query(self, offset=0, limit=10, resulttype='results', + datetime_=None, properties=[], sortby=[], + select_properties=[], q=None, filterq=None, **kwargs): + """ + Query Postgresql for all the content. + e,g: http://localhost:5000/collections/hotosm_bdi_waterways/items? + limit=1&resulttype=results + + :param offset: starting record to return (default 0) + :param limit: number of records to return (default 10) + :param resulttype: return results or hit limit (default results) + :param datetime_: temporal (datestamp or extent) + :param properties: list of tuples (name, value) + :param sortby: list of dicts (property, order) + :param select_properties: list of property names + :param q: full-text search term(s) + + :returns: JSON array + """ + + LOGGER.debug('Preparing filters') + property_filters = self._get_property_filters(properties) + order_by_clauses = self._get_order_by_clauses(sortby, self.table_model) + selected_properties = self._select_properties_clause(select_properties) + + LOGGER.debug('Querying PostgreSQL') + # Execute query within self-closing database Session context + with Session(self._engine) as session: + results = (session.query(self.table_model) + .filter(property_filters) + .order_by(*order_by_clauses) + .options(selected_properties) + .offset(offset)) + + matched = results.count() + if limit < matched: + returned = limit + else: + returned = matched + + LOGGER.debug(f'Found {matched} result(s)') + + LOGGER.debug('Preparing response') + response = { + 'type': 'Vocabulary', + 'items': [], + 'numberMatched': matched, + 'numberReturned': returned + } + + if resulttype == "hits" or not results: + response['numberReturned'] = 0 + return response + + for item in results.limit(limit): + response['items'].append(self._sqlalchemy_to_dict(item)) + + return response + + def get_fields(self): + """ + Return fields (columns) from PostgreSQL table + + :returns: dict of fields + """ + LOGGER.debug('Get available fields/properties') + + fields = {} + for column in self.table_model.__table__.columns: + fields[str(column.name)] = {'type': str(column.type)} + + return fields + + def get(self, identifier, **kwargs): + """ + Query the provider for a specific + feature id e.g: /collections/hotosm_bdi_waterways/items/13990765 + + :param identifier: feature id + + :returns: JSON Array + """ + LOGGER.debug(f'Get item by ID: {identifier}') + + # Execute query within self-closing database Session context + with Session(self._engine) as session: + # Retrieve data from database as feature + query = session.query(self.table_model) + item = query.get(identifier) + if item is None: + msg = f"No such item: {self.id_field}={identifier}." + raise ProviderItemNotFoundError(msg) + + feature = self._sqlalchemy_to_dict(item) + + LOGGER.debug(feature) + + # Drop non-defined properties + if self.properties: + props = feature['properties'] + dropping_keys = deepcopy(props).keys() + for item in dropping_keys: + if item not in self.properties: + props.pop(item) + + # Add fields for previous and next items + id_field = getattr(self.table_model, self.id_field) + prev_item = (session.query(self.table_model) + .order_by(id_field.desc()) + .filter(id_field < identifier) + .first()) + next_item = (session.query(self.table_model) + .order_by(id_field.asc()) + .filter(id_field > identifier) + .first()) + feature['prev'] = (getattr(prev_item, self.id_field) + if prev_item is not None else identifier) + feature['next'] = (getattr(next_item, self.id_field) + if next_item is not None else identifier) + + return feature + + def _store_db_parameters(self, parameters): + self.db_user = parameters.get('user') + self.db_host = parameters.get('host') + self.db_port = parameters.get('port', 5432) + self.db_name = parameters.get('dbname') + self.db_search_path = parameters.get('search_path', ['public']) + self._db_password = parameters.get('password') + + def _get_engine_and_table_model(self): + """ + Create a SQL Alchemy engine for the database and reflect the table + model. Use existing versions from stores if available to allow reuse + of Engine connection pool and save expensive table reflection. + """ + # One long-lived engine is used per database URL: + # https://docs.sqlalchemy.org/en/14/core/connections.html#basic-usage + engine_store_key = (self.db_user, self.db_host, self.db_port, + self.db_name) + try: + engine = _ENGINE_STORE[engine_store_key] + except KeyError: + conn_str = ( + 'postgresql+psycopg2://' + f'{self.db_user}:{self._db_password}@' + f'{self.db_host}:{self.db_port}/' + f'{self.db_name}' + ) + engine = create_engine( + conn_str, + connect_args={'client_encoding': 'utf8', + 'application_name': 'pygeoapi'}, + pool_pre_ping=True) + _ENGINE_STORE[engine_store_key] = engine + + # Reuse table model if one exists + table_model_store_key = (self.db_host, self.db_port, self.db_name, + self.table) + try: + table_model = _TABLE_MODEL_STORE[table_model_store_key] + except KeyError: + table_model = self._reflect_table_model(engine) + _TABLE_MODEL_STORE[table_model_store_key] = table_model + + return engine, table_model + + def _reflect_table_model(self, engine): + """ + Reflect database metadata to create a SQL Alchemy model corresponding + to target table. This requires a database query and is expensive to + perform. + """ + metadata = MetaData(engine) + + # Look for table in the first schema in the search path + try: + schema = self.db_search_path[0] + metadata.reflect(schema=schema, only=[self.table], views=True) + except OperationalError: + msg = (f"Could not connect to {repr(engine.url)} " + "(password hidden).") + raise ProviderConnectionError(msg) + except InvalidRequestError: + msg = (f"Table '{self.table}' not found in schema '{schema}' " + f"on {repr(engine.url)}.") + raise ProviderQueryError(msg) + + # Create SQLAlchemy model from reflected table + # It is necessary to add the primary key constraint because SQLAlchemy + # requires it to reflect the table, but a view in a PostgreSQL database + # does not have a primary key defined. + sqlalchemy_table_def = metadata.tables[f'{schema}.{self.table}'] + try: + sqlalchemy_table_def.append_constraint( + PrimaryKeyConstraint(self.id_field) + ) + except KeyError: + msg = (f"No such id_field column ({self.id_field}) on " + f"{schema}.{self.table}.") + raise ProviderQueryError(msg) + + Base = automap_base(metadata=metadata) + Base.prepare() + TableModel = getattr(Base.classes, self.table) + + return TableModel + + def _sqlalchemy_to_dict(self, item): + + # Add properties from item + item_dict = item.__dict__ + item_dict.pop('_sa_instance_state') # Internal SQLAlchemy metadata + return item_dict + + def _get_order_by_clauses(self, sort_by, table_model): + # Build sort_by clauses if provided + clauses = [] + for sort_by_dict in sort_by: + model_column = getattr(table_model, sort_by_dict['property']) + order_function = asc if sort_by_dict['order'] == '+' else desc + clauses.append(order_function(model_column)) + + # Otherwise sort by primary key (to ensure reproducible output) + if not clauses: + clauses.append(asc(getattr(table_model, self.id_field))) + + return clauses + + def _get_property_filters(self, properties): + if not properties: + return True # Let everything through + + # Convert property filters into SQL Alchemy filters + # Based on https://stackoverflow.com/a/14887813/3508733 + filter_group = [] + for column_name, value in properties: + column = getattr(self.table_model, column_name) + filter_group.append(column == value) + property_filters = and_(*filter_group) + + return property_filters + + def _select_properties_clause(self, select_properties): + # List the column names that we want + if select_properties: + column_names = set(select_properties) + else: + column_names = set(self.fields.keys()) + + if self.properties: # optional subset of properties defined in config + properties_from_config = set(self.properties) + column_names = column_names.intersection(properties_from_config) + + # Convert names to SQL Alchemy clause + selected_columns = [] + for column_name in column_names: + try: + column = getattr(self.table_model, column_name) + selected_columns.append(column) + except AttributeError: + pass # Ignore non-existent columns + selected_properties_clause = load_only(*selected_columns) + + return selected_properties_clause diff --git a/pygeoapi/templates/collections/collection.html b/pygeoapi/templates/collections/collection.html index 61288cbe8..3ba5e1fe3 100644 --- a/pygeoapi/templates/collections/collection.html +++ b/pygeoapi/templates/collections/collection.html @@ -30,6 +30,21 @@

{{ data['title'] }}

+ {% set ns = namespace(header_printed=false) %} + {% for link in data['links'] %} + {% if link['rel'] == 'license' %} + {% if not ns.header_printed %} +

{% trans %}License{% endtrans %}

+ {% set ns.header_printed = true %} + {% endif %} + + {% endif %} + {% endfor %} {% if data['itemType'] == 'feature' or data['itemType'] == 'record' %}

{% trans %}Browse{% endtrans %}

    diff --git a/pygeoapi/templates/landing_page.html b/pygeoapi/templates/landing_page.html index 7bc0d2b78..284aac0aa 100644 --- a/pygeoapi/templates/landing_page.html +++ b/pygeoapi/templates/landing_page.html @@ -49,6 +49,12 @@

    {% trans %}Collections{% endtrans %}

    {% trans %}View the collections in this service{% endtrans %}

    +
    +

    {% trans %}Vocabularies{% endtrans %}

    +

    + {% trans %}View the vocabularies in this service{% endtrans %} +

    +
    {% if data['stac'] %}

    {% trans %}SpatioTemporal Assets{% endtrans %}

    diff --git a/pygeoapi/templates/vocabularies/index.html b/pygeoapi/templates/vocabularies/index.html new file mode 100644 index 000000000..f0ff2dfad --- /dev/null +++ b/pygeoapi/templates/vocabularies/index.html @@ -0,0 +1,35 @@ +{% extends "_base.html" %} +{% block title %}{{ super() }} {% trans %}Vocabularies{% endtrans %} {% endblock %} +{% block crumbs %}{{ super() }} +/ {% trans %}Vocabularies{% endtrans %} +{% endblock %} +{% block body %} +
    +

    {% trans %}Vocabularies in this service{% endtrans %}

    +
    +
    + + + + + + + + + {% for vcb in data['vocabularies'] %} + + + + + {% endfor %} + +
    {% trans %}Name{% endtrans %}{% trans %}Description{% endtrans %}
    + + {{ vcb['title'] | striptags | truncate }} + + {{ vcb['description'] | striptags | truncate }} +
    +
    +
    +
    +{% endblock %} diff --git a/pygeoapi/templates/vocabularies/items/index.html b/pygeoapi/templates/vocabularies/items/index.html new file mode 100644 index 000000000..cef2b0d71 --- /dev/null +++ b/pygeoapi/templates/vocabularies/items/index.html @@ -0,0 +1,91 @@ +{% extends "_base.html" %} +{% block title %}{{ super() }} {{ data['title'] }} {% endblock %} +{% block crumbs %}{{ super() }} +/ {% trans %}Vocabularies{% endtrans %} +{% for link in data['links'] %} + {% if link.rel == 'vocabulary' %} / + {{ link['title'] | string | truncate( 25 ) }} + {% set col_title = link['title'] %} + {% endif %} +{% endfor %} +/ {% trans %}Items{% endtrans %} +{% endblock %} + +{% block body %} +
    +
    +

    {% for l in data['links'] if l.rel == 'vocabulary' %} {{ l['title'] }} {% endfor %}

    +

    {% trans %}Items in this vocabulary{% endtrans %}.

    +
    +
    + {% if data['items'] %} +
    +
    + {% set props = [] %} + + + + {% if data.get('uri_field') %} + {% set uri_field = data.uri_field %} + + {% elif data.get('title_field') %} + {% set title_field = data.title_field %} + + {% else %} + + {% endif %} + + {% for k in data['items'][0].keys() %} + {% if k not in [data.id_field, data.title_field, data.uri_field] %} + {% set props = props.append(k) %} + + {% endif %} + {% endfor %} + + + + + {% for ft in data['items'] %} + + {% if data.get('uri_field') %} + {% set uri_field = data.uri_field %} + + {% elif data.get('title_field') %} + {% set title_field = data.title_field %} + + {% else %} + + {% endif %} + + {% for prop in props %} + + {% endfor %} + + + {% endfor %} + +
    {{ uri_field }}{{ title_field }}id{{ k }}
    + + {{ ft.get(uri_field) }} + + + + {{ ft.get(title_field) | string | truncate( 35 ) }} + + + + {{ ft.id | string | truncate( 12 ) }} + + + {{ ft.get(prop, '') | string | truncate( 35 ) }} +
    +
    +
    + {% else %} +
    +

    {% trans %}No items{% endtrans %}

    +
    + {% endif %} +
    +{% endblock %} + diff --git a/pygeoapi/templates/vocabularies/items/item.html b/pygeoapi/templates/vocabularies/items/item.html new file mode 100644 index 000000000..5b420753d --- /dev/null +++ b/pygeoapi/templates/vocabularies/items/item.html @@ -0,0 +1,93 @@ +{% extends "_base.html" %} +{% set ptitle = data[data['title_field']] or 'Item '.format(data['id']) %} +{% block desc %}{{ data.get('description', {}) | string | truncate(250) }}{% endblock %} +{% block tags %}{{ data.get('themes', [{}])[0].get('concepts', []) | join(',') }}{% endblock %} +{# Optionally renders an img element, otherwise standard value or link rendering #} +{% macro render_item_value(v, width) -%} + {% set val = v | string | trim %} + {% if val|length and val.lower().endswith(('.jpg', '.jpeg', '.png', '.gif', '.bmp')) %} + {# Ends with image extension: render img element with link to image #} + {{ val.split('/') | last }} + {% elif v is string or v is number %} + {{ val | urlize() }} + {% elif v is mapping %} + {% for i,j in v.items() %} + {{ i }}: {{ render_item_value(j, 60) }}
    + {% endfor %} + {% elif v is iterable %} + {% for i in v %} + {{ render_item_value(i, 60) }} + {% endfor %} + {% else %} + {{ val | urlize() }} + {% endif %} +{%- endmacro %} +{% block title %}{{ ptitle }}{% endblock %} +{% block crumbs %}{{ super() }} +/ {% trans %}Vocabularies{% endtrans %} +{% for link in data['links'] %} + {% if link.rel == 'vocabulary' %} +/ {{ link['title'] | truncate( 25 ) }} + {% endif %} +{% endfor %} +/ {% trans %}Items{% endtrans %} +/ {{ ptitle | truncate( 25 ) }} +{% endblock %} + +{% block body %} +
    +
    +
    +

    {{ ptitle }}

    +
    +
    +
    +
    + + + + + + + + + {% if data.uri_field %} + + + + + {% endif %} + + + + + {% for k, v in data.items() %} + {% if k != data['id_field'] %} + + + + + {% endif %} + {% endfor %} + + + + + +
    {% trans %}Property{% endtrans %}{% trans %}Value{% endtrans %}
    {{ data.uri_field }}{{ data['properties'].pop(data.uri_field) }}
    id{{ data.id }}
    {{ k }}{{ render_item_value(v, 80) }}
    Links + +
    +
    +
    +
    +{% endblock %} diff --git a/pygeoapi/templates/vocabularies/vocabulary.html b/pygeoapi/templates/vocabularies/vocabulary.html new file mode 100644 index 000000000..82685b928 --- /dev/null +++ b/pygeoapi/templates/vocabularies/vocabulary.html @@ -0,0 +1,65 @@ +{% extends "_base.html" %} +{% block title %}{{ super() }} {{ data['title'] }} {% endblock %} +{% block desc %}{{ data.get('description','') | truncate(250) }}{% endblock %} +{% block tags %}{{ data.get('keywords',[]) | join(',') }}{% endblock %} +{% block crumbs %}{{ super() }} +/ {% trans %}Vocabularies{% endtrans %} +/ {{ data['title'] | truncate( 25 ) }} +{% endblock %} + +{% block body %} +
    +
    +
    +

    {{ data['title'] }}

    +

    {{ data['description'] }}

    +

    + {% for kw in data['keywords'] %} + {{ kw }} + {% endfor %} +

    +
    +
    + {{ data }} + {% if data['itemType'] == 'Vocabulary' or data['itemType'] == 'record' %} +

    {% trans %}Browse{% endtrans %}

    + +

    {% trans %}Queryables{% endtrans %}

    + + {% for provider in config['resources'][data['id']]['providers'] %} + {% if 'tile' in provider['type'] %} +

    {% trans %}Tiles{% endtrans %}

    + + {% endif %} + {% endfor %} + {% endif %} +

    {% trans %}Links{% endtrans %}

    + +
    +{% endblock %} + diff --git a/requirements.txt b/requirements.txt index c467e50ec..69810e97e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ pytz PyYAML rasterio requests -shapely<2.0 +shapely SQLAlchemy<2.0.0 tinydb unicodecsv \ No newline at end of file From d455aef571784d868ef6b75b7789ae8bfb50d09c Mon Sep 17 00:00:00 2001 From: david-i-berry Date: Thu, 27 Apr 2023 17:19:51 +0200 Subject: [PATCH 2/5] requirements.txt - click and jinja2 versions. --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 69810e97e..281743042 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ Babel -click>7,<=8 +click Flask -jinja2==3.0.3 +jinja2 jsonschema pydantic pygeofilter From 88dbfb191b120592ea375a262b8905ad84c96a81 Mon Sep 17 00:00:00 2001 From: david-i-berry Date: Thu, 27 Apr 2023 17:26:28 +0200 Subject: [PATCH 3/5] jinja2 - autoescape fix --- pygeoapi/util.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pygeoapi/util.py b/pygeoapi/util.py index a877be9f4..651042258 100644 --- a/pygeoapi/util.py +++ b/pygeoapi/util.py @@ -412,8 +412,7 @@ def render_j2_template(config: dict, template: Path, LOGGER.debug(f'using default templates: {TEMPLATES}') env = Environment(loader=FileSystemLoader(template_paths), - extensions=['jinja2.ext.i18n', - 'jinja2.ext.autoescape'], + extensions=['jinja2.ext.i18n'], autoescape=select_autoescape(['html', 'xml'])) env.filters['to_json'] = to_json From 6b9ad351b062f6652052eca7eaf125822fc6b902 Mon Sep 17 00:00:00 2001 From: david-i-berry Date: Wed, 10 May 2023 14:11:58 +0200 Subject: [PATCH 4/5] addition of datetime filter for psql. --- pygeoapi/provider/postgresql.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/pygeoapi/provider/postgresql.py b/pygeoapi/provider/postgresql.py index e861812e0..47c8723cc 100644 --- a/pygeoapi/provider/postgresql.py +++ b/pygeoapi/provider/postgresql.py @@ -99,11 +99,13 @@ def __init__(self, provider_def): self.table = provider_def['table'] self.id_field = provider_def['id_field'] self.geom = provider_def.get('geom_field', 'geom') + self.time_field = provider_def.get('time_field', 'datetime') LOGGER.debug(f'Name: {self.name}') LOGGER.debug(f'Table: {self.table}') LOGGER.debug(f'ID field: {self.id_field}') LOGGER.debug(f'Geometry field: {self.geom}') + LOGGER.debug(f'Time field: {self.time_field}') # Read table information from database self._store_db_parameters(provider_def['data']) @@ -140,10 +142,12 @@ def query(self, offset=0, limit=10, resulttype='results', property_filters = self._get_property_filters(properties) cql_filters = self._get_cql_filters(filterq) bbox_filter = self._get_bbox_filter(bbox) + time_filter = self._get_datetime_filter(datetime_) order_by_clauses = self._get_order_by_clauses(sortby, self.table_model) selected_properties = self._select_properties_clause(select_properties, skip_geometry) + LOGGER.debug(selected_properties) LOGGER.debug('Querying PostGIS') # Execute query within self-closing database Session context with Session(self._engine) as session: @@ -151,6 +155,7 @@ def query(self, offset=0, limit=10, resulttype='results', .filter(property_filters) .filter(cql_filters) .filter(bbox_filter) + .filter(time_filter) .order_by(*order_by_clauses) .options(selected_properties) .offset(offset)) @@ -514,6 +519,29 @@ def _get_bbox_filter(self, bbox): return bbox_filter + def _get_datetime_filter(self, datetime_): + + if datetime_ is None: + LOGGER.debug(True) + return True + else: + LOGGER.debug('processing datetime parameter') + if self.time_field is None: + LOGGER.error('time_field not enabled for collection') + raise ProviderQueryError() + + time_field = self.time_field + time_column = geom_column = getattr(self.table_model, time_field) + + if '/' in datetime_: # envelope + LOGGER.debug('detected time range') + time_begin, time_end = datetime_.split('/') + filter = time_column.between(time_begin, time_end) + else: + filter = time_column == datetime_ + LOGGER.debug(filter) + return filter + def _select_properties_clause(self, select_properties, skip_geometry): # List the column names that we want if select_properties: From 18578acc8734d3bbab9cc0d5d27036d0f6611752 Mon Sep 17 00:00:00 2001 From: david-i-berry Date: Sun, 14 May 2023 15:45:44 +0200 Subject: [PATCH 5/5] Logger debug feedback added. --- pygeoapi/provider/postgresql.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pygeoapi/provider/postgresql.py b/pygeoapi/provider/postgresql.py index 47c8723cc..774a8301d 100644 --- a/pygeoapi/provider/postgresql.py +++ b/pygeoapi/provider/postgresql.py @@ -272,6 +272,7 @@ def create(self, item): session.execute(statement) session.commit() + LOGGER.debug( f"feature {feature['id']} added") return feature['id'] def update(self, identifier, item): @@ -465,7 +466,13 @@ def _feature_to_sqlalchemy(self, feature): result[self.geom] = f"SRID=4326;{geom}" for column in self.table_model.__table__.columns: if column.name not in (self.id_field, self.geom): - result[column.name] = feature['properties'][column.name] + if feature['properties'][column.name] not in (None, ''): + result[column.name] = feature['properties'][column.name] + else: + result[column.name] = None + + LOGGER.debug(result) + return result def _get_order_by_clauses(self, sort_by, table_model):