diff --git a/geonode/base/api/deprecated_extra_metadata.py b/geonode/base/api/deprecated_extra_metadata.py new file mode 100644 index 00000000000..e0d3ec942b8 --- /dev/null +++ b/geonode/base/api/deprecated_extra_metadata.py @@ -0,0 +1,346 @@ +# ######################################################################### +# +# Copyright (C) 2025 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# ######################################################################### +""" +Deprecated backward-compatible adapters for the ExtraMetadata API. + +These adapters re-expose the old ``/extra_metadata/`` endpoint and the +``metadata`` serializer field using :class:`SparseField` as the storage +backend. They are marked **deprecated** and should be removed after the +deprecation period. + +To remove these adapters: + 1. Delete this file. + 2. Remove the ``metadata`` field and import from ``serializers.py``. + 3. Remove ``DeprecatedExtraMetadataMixin`` and its import from ``views.py``. + 4. Remove deprecated settings from ``settings.py`` + (``DEFAULT_EXTRA_METADATA_SCHEMA``, ``CUSTOM_METADATA_SCHEMA``, + ``EXTRA_METADATA_SCHEMA``, and the ``from schema import Optional`` import). +""" + +import json +import logging +import uuid +import warnings + +from deprecated import deprecated +from rest_framework.decorators import action +from rest_framework.response import Response +from drf_spectacular.utils import extend_schema + +from dynamic_rest.fields.fields import DynamicComputedField + +from django.db import IntegrityError + +from geonode.base.models import ResourceBase +from geonode.metadata.models import SparseField + +logger = logging.getLogger(__name__) + +DEPRECATION_VERSION = "4.4.0" +DEPRECATION_REASON = ( + "The extra_metadata API is deprecated and will be removed in a future " + "version. Use the sparse fields API instead." +) +SPARSE_FIELD_PREFIX = "extra_metadata_" +# SparseField.value is CharField(max_length=1024); entries exceeding this +# limit cannot be stored and are silently skipped with a log warning. +SPARSE_FIELD_VALUE_MAX_LENGTH = 1024 + +# Query parameter names that should *not* be treated as legacy +# ``metadata__`` filters in the deprecated GET endpoint. +_NON_FILTER_QUERY_PARAMS = { + "api_preset", "page", "page_size", "format", "include[]", + "exclude[]", "sort[]", +} + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _sparse_fields_for_resource(resource): + """Return SparseField entries that represent migrated ExtraMetadata.""" + return SparseField.objects.filter( + resource=resource, + name__startswith=SPARSE_FIELD_PREFIX, + ) + + +def _sparse_to_legacy(sparse_field): + """Convert a SparseField into the old ExtraMetadata representation. + + Returns ``{"id": , ...metadata_dict}`` so existing consumers see the + same shape they used to get. + """ + try: + metadata = json.loads(sparse_field.value) if sparse_field.value else {} + except (json.JSONDecodeError, TypeError): + metadata = {} + return {**{"id": sparse_field.pk}, **metadata} + + +def _next_sparse_name(resource): + """Generate the next available ``extra_metadata_`` name. + + Falls back to a UUID-based suffix if a name collision occurs (e.g. under + concurrent requests), keeping the ``extra_metadata_`` prefix intact. + The numeric part is still preferred for readability. + """ + existing = ( + SparseField.objects.filter( + resource=resource, + name__startswith=SPARSE_FIELD_PREFIX, + ) + .order_by("-name") + .values_list("name", flat=True) + ) + max_n = 0 + for name in existing: + suffix = name[len(SPARSE_FIELD_PREFIX):] + try: + max_n = max(max_n, int(suffix)) + except (ValueError, TypeError): + pass + candidate = f"{SPARSE_FIELD_PREFIX}{max_n + 1}" + # Guard against a race: if the candidate already exists, use a UUID suffix + if SparseField.objects.filter(resource=resource, name=candidate).exists(): + candidate = f"{SPARSE_FIELD_PREFIX}{uuid.uuid4().hex[:12]}" + return candidate + + +# --------------------------------------------------------------------------- +# Deprecated serializer field (for ``metadata`` on ResourceBaseSerializer) +# --------------------------------------------------------------------------- + + +class DeprecatedExtraMetadataField(DynamicComputedField): + """Deferred computed field that reconstructs the legacy ``metadata`` + representation from :class:`SparseField` entries. + + .. deprecated:: 4.4.0 + Use the sparse fields API instead. + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + @deprecated(version=DEPRECATION_VERSION, reason=DEPRECATION_REASON) + def get_attribute(self, instance): + warnings.warn(DEPRECATION_REASON, DeprecationWarning, stacklevel=2) + logger.warning(DEPRECATION_REASON) + try: + qs = _sparse_fields_for_resource(instance) + return [_sparse_to_legacy(sf) for sf in qs] + except Exception as e: + logger.exception(e) + return [] + + +# --------------------------------------------------------------------------- +# Deprecated extra_metadata view action mixin +# --------------------------------------------------------------------------- + + +class DeprecatedExtraMetadataMixin: + """Mixin that adds the deprecated ``extra_metadata`` action back to a + ``ViewSet``. + + Import this mixin and add it to your ViewSet's bases to restore the + old ``/{pk}/extra_metadata/`` endpoint backed by SparseFields. + + .. deprecated:: 4.4.0 + Use the sparse fields API instead. + """ + + @extend_schema( + methods=["get", "put", "delete", "post"], + description=( + "[DEPRECATED] Get/Update/Delete/Add extra metadata for a resource. " + "Use the sparse fields API instead." + ), + deprecated=True, + ) + @action( + detail=True, + methods=["get", "put", "delete", "post"], + url_path=r"extra_metadata", + url_name="extra-metadata", + ) + def extra_metadata(self, request, pk, *args, **kwargs): + """Deprecated endpoint – delegates to SparseField storage.""" + warnings.warn(DEPRECATION_REASON, DeprecationWarning, stacklevel=2) + logger.warning(DEPRECATION_REASON) + + resource = ResourceBase.objects.filter(pk=pk).first() + if resource is None: + return Response({"detail": "Not found."}, status=404) + + if request.method == "GET": + return self._extra_metadata_get(request, resource) + elif request.method == "POST": + return self._extra_metadata_post(request, resource) + elif request.method == "PUT": + return self._extra_metadata_put(request, resource) + elif request.method == "DELETE": + return self._extra_metadata_delete(request, resource) + + # -- GET ---------------------------------------------------------------- + + @staticmethod + def _extra_metadata_get(request, resource): + qs = _sparse_fields_for_resource(resource) + # Support the old query-param filtering (e.g. ?field_name=value) + for key, value in request.query_params.items(): + if key in _NON_FILTER_QUERY_PARAMS or key.startswith("filter{"): + continue + # Old API used metadata__=value JSONField lookups. + # We approximate this by filtering on the JSON string. + filtered = [] + for sf in qs: + try: + meta = json.loads(sf.value) if sf.value else {} + except (json.JSONDecodeError, TypeError): + continue + if str(meta.get(key)) == str(value): + filtered.append(sf) + qs = filtered + break # Old API only used the first filter pair + + if isinstance(qs, list): + return Response([_sparse_to_legacy(sf) for sf in qs]) + return Response([_sparse_to_legacy(sf) for sf in qs.iterator()]) + + # -- POST --------------------------------------------------------------- + + @staticmethod + def _extra_metadata_post(request, resource): + data = request.data + if isinstance(data, str): + try: + data = json.loads(data) + except (json.JSONDecodeError, TypeError): + return Response( + {"detail": "Invalid JSON payload."}, + status=400, + ) + + if not isinstance(data, list): + return Response( + {"detail": "Expected a JSON list of metadata objects."}, + status=400, + ) + + for meta_dict in data: + if not isinstance(meta_dict, dict): + continue + meta_dict.pop("id", None) + value = json.dumps(meta_dict) + if len(value) > SPARSE_FIELD_VALUE_MAX_LENGTH: + logger.warning( + "extra_metadata entry skipped: serialized value exceeds " + f"{SPARSE_FIELD_VALUE_MAX_LENGTH} characters" + ) + continue + name = _next_sparse_name(resource) + try: + SparseField.objects.create( + resource=resource, name=name, value=value + ) + except IntegrityError: + # Concurrent request created the same name; retry with UUID + name = f"{SPARSE_FIELD_PREFIX}{uuid.uuid4().hex[:12]}" + SparseField.objects.create( + resource=resource, name=name, value=value + ) + + result = [_sparse_to_legacy(sf) for sf in _sparse_fields_for_resource(resource)] + return Response(result, status=201) + + # -- PUT ---------------------------------------------------------------- + + @staticmethod + def _extra_metadata_put(request, resource): + data = request.data + if isinstance(data, str): + try: + data = json.loads(data) + except (json.JSONDecodeError, TypeError): + return Response( + {"detail": "Invalid JSON payload."}, + status=400, + ) + + if not isinstance(data, list): + return Response( + {"detail": "Expected a JSON list of metadata objects."}, + status=400, + ) + + for meta_dict in data: + if not isinstance(meta_dict, dict): + continue + sf_id = meta_dict.pop("id", None) + if sf_id is None: + continue + value = json.dumps(meta_dict) + if len(value) > SPARSE_FIELD_VALUE_MAX_LENGTH: + logger.warning( + "extra_metadata entry skipped: serialized value exceeds " + f"{SPARSE_FIELD_VALUE_MAX_LENGTH} characters" + ) + continue + SparseField.objects.filter( + pk=sf_id, + resource=resource, + name__startswith=SPARSE_FIELD_PREFIX, + ).update(value=value) + + result = [_sparse_to_legacy(sf) for sf in _sparse_fields_for_resource(resource)] + return Response(result) + + # -- DELETE ------------------------------------------------------------- + + @staticmethod + def _extra_metadata_delete(request, resource): + data = request.data + if isinstance(data, str): + try: + data = json.loads(data) + except (json.JSONDecodeError, TypeError): + return Response( + {"detail": "Invalid JSON payload."}, + status=400, + ) + + if not isinstance(data, list): + return Response( + {"detail": "Expected a JSON list of IDs."}, + status=400, + ) + + ids = [int(i) for i in data if isinstance(i, (int, str)) and str(i).isdigit()] + SparseField.objects.filter( + pk__in=ids, + resource=resource, + name__startswith=SPARSE_FIELD_PREFIX, + ).delete() + + result = [_sparse_to_legacy(sf) for sf in _sparse_fields_for_resource(resource)] + return Response(result) diff --git a/geonode/base/api/serializers.py b/geonode/base/api/serializers.py index 812b7e98ac3..6d8094518d0 100644 --- a/geonode/base/api/serializers.py +++ b/geonode/base/api/serializers.py @@ -67,6 +67,7 @@ from geonode.assets.handlers import asset_handler_registry from geonode.utils import build_absolute_uri from geonode.security.utils import get_resources_with_perms, get_geoapp_subtypes +from geonode.base.api.deprecated_extra_metadata import DeprecatedExtraMetadataField from geonode.resource.models import ExecutionRequest from django.contrib.gis.geos import Polygon from geonode.security.registry import permissions_registry @@ -654,6 +655,8 @@ class ResourceBaseSerializer(MultiLangOutputMixin, DynamicModelSerializer): links = DynamicRelationField(LinksSerializer, source="id", read_only=True) # Deferred fields + # Deprecated: use the sparse fields API instead of ``metadata``. + metadata = DeprecatedExtraMetadataField(deferred=True, read_only=True) data = DataBlobField(DataBlobSerializer, source="id", deferred=True, required=False) executions = DynamicRelationField( ResourceExecutionRequestSerializer, source="id", deferred=True, required=False, read_only=True @@ -735,6 +738,7 @@ class Meta: "sourcetype", "is_copyable", "blob", + "metadata", # Deprecated: use sparse fields API instead "executions", "linked_resources", "download_url", diff --git a/geonode/base/api/tests.py b/geonode/base/api/tests.py index f2bc56fb1bb..db75190e591 100644 --- a/geonode/base/api/tests.py +++ b/geonode/base/api/tests.py @@ -4175,3 +4175,152 @@ def test_map_layer_permission_caching(self): # Check that the permissions in the layers are the same for layer1, layer2 in zip(data1["map"]["maplayers"], data2["map"]["maplayers"]): self.assertEqual(layer1["dataset"]["perms"], layer2["dataset"]["perms"]) + + +class DeprecatedExtraMetadataApiTest(GeoNodeBaseTestSupport): + """Tests for the deprecated backward-compatible ExtraMetadata API adapters. + + These adapters re-expose the old ``/extra_metadata/`` endpoint and the + ``metadata`` serializer field using SparseField as the storage backend. + """ + + def setUp(self): + super().setUp() + self.admin = get_user_model().objects.get(username="admin") + self.dataset = create_single_dataset("deprecated_em_test") + + def _url(self, pk=None): + pk = pk or self.dataset.pk + return urljoin( + f"{reverse('base-resources-list')}/", + f"{pk}/extra_metadata/", + ) + + def test_get_empty_extra_metadata(self): + """GET should return an empty list when no extra metadata exists.""" + self.client.login(username="admin", password="admin") + response = self.client.get(self._url()) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json(), []) + + def test_post_extra_metadata(self): + """POST should create new extra metadata entries.""" + from geonode.metadata.models import SparseField + + self.client.login(username="admin", password="admin") + payload = [ + {"field_name": "test_field", "field_value": "test_value"}, + {"field_name": "another", "field_value": "value2"}, + ] + response = self.client.post( + self._url(), + data=json.dumps(payload), + content_type="application/json", + ) + self.assertEqual(response.status_code, 201) + result = response.json() + self.assertEqual(len(result), 2) + # Each entry should have an id and the metadata fields + for item in result: + self.assertIn("id", item) + self.assertIn("field_name", item) + self.assertIn("field_value", item) + + # Verify SparseField entries were created + sf_count = SparseField.objects.filter( + resource=self.dataset.resourcebase_ptr, + name__startswith="extra_metadata_", + ).count() + self.assertEqual(sf_count, 2) + + def test_get_returns_posted_metadata(self): + """GET after POST should return the created metadata.""" + self.client.login(username="admin", password="admin") + payload = [{"field_name": "myfield", "field_value": "myvalue"}] + self.client.post( + self._url(), + data=json.dumps(payload), + content_type="application/json", + ) + response = self.client.get(self._url()) + self.assertEqual(response.status_code, 200) + result = response.json() + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["field_name"], "myfield") + self.assertEqual(result[0]["field_value"], "myvalue") + + def test_put_updates_existing_metadata(self): + """PUT should update an existing entry by id.""" + self.client.login(username="admin", password="admin") + # Create first + payload = [{"field_name": "original", "field_value": "v1"}] + response = self.client.post( + self._url(), + data=json.dumps(payload), + content_type="application/json", + ) + created_id = response.json()[0]["id"] + + # Update + update_payload = [ + {"id": created_id, "field_name": "updated", "field_value": "v2"} + ] + response = self.client.put( + self._url(), + data=json.dumps(update_payload), + content_type="application/json", + ) + self.assertEqual(response.status_code, 200) + result = response.json() + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["field_name"], "updated") + self.assertEqual(result[0]["field_value"], "v2") + + def test_delete_removes_metadata(self): + """DELETE should remove entries by id.""" + self.client.login(username="admin", password="admin") + # Create + payload = [ + {"field_name": "to_delete", "field_value": "val"}, + {"field_name": "to_keep", "field_value": "keep"}, + ] + response = self.client.post( + self._url(), + data=json.dumps(payload), + content_type="application/json", + ) + items = response.json() + delete_id = items[0]["id"] + + # Delete one + response = self.client.delete( + self._url(), + data=json.dumps([delete_id]), + content_type="application/json", + ) + self.assertEqual(response.status_code, 200) + result = response.json() + self.assertEqual(len(result), 1) + self.assertNotEqual(result[0]["id"], delete_id) + + def test_metadata_field_in_serializer(self): + """The deprecated ``metadata`` field should appear when requested + via include[] and return data from SparseField entries.""" + from geonode.metadata.models import SparseField + + self.client.login(username="admin", password="admin") + SparseField.objects.create( + resource=self.dataset.resourcebase_ptr, + name="extra_metadata_1", + value=json.dumps({"field_name": "test", "field_value": "val"}), + ) + url = f"{reverse('base-resources-list')}/{self.dataset.pk}?include[]=metadata" + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + data = response.json().get("resource", response.json()) + self.assertIn("metadata", data) + metadata = data["metadata"] + self.assertIsInstance(metadata, list) + if metadata: + self.assertIn("id", metadata[0]) + self.assertIn("field_name", metadata[0]) diff --git a/geonode/base/api/views.py b/geonode/base/api/views.py index 6ba8bfcf463..253d27f3083 100644 --- a/geonode/base/api/views.py +++ b/geonode/base/api/views.py @@ -110,6 +110,7 @@ from geonode.people.api.serializers import UserSerializer from .pagination import GeoNodeApiPagination from geonode.base.utils import patch_perms +from geonode.base.api.deprecated_extra_metadata import DeprecatedExtraMetadataMixin from geonode.assets.models import Asset from geonode.assets.utils import create_asset_and_link, unlink_asset from geonode.assets.handlers import asset_handler_registry @@ -303,7 +304,9 @@ def replace_presets(self, request): request.GET._mutable = False -class ResourceBaseViewSet(ApiPresetsInitializer, MultiLangViewMixin, DynamicModelViewSet): +class ResourceBaseViewSet( + ApiPresetsInitializer, MultiLangViewMixin, DeprecatedExtraMetadataMixin, DynamicModelViewSet +): """ API endpoint that allows base resources to be viewed or edited. """ diff --git a/geonode/settings.py b/geonode/settings.py index 8fc7ff81315..b021f2d038b 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -25,6 +25,7 @@ import logging import subprocess import dj_database_url +from schema import Optional from urllib.parse import urlparse, urljoin # @@ -2068,6 +2069,32 @@ def get_geonode_catalogue_service(): DEFAULT_AUTO_FIELD = "django.db.models.AutoField" +# ---- Deprecated ExtraMetadata settings (kept for backward compatibility) ---- +# These settings are deprecated and will be removed in a future version. +# The ExtraMetadata model has been replaced by SparseField. +# External clients (e.g. mapstore-client) may still reference these settings. + +DEFAULT_EXTRA_METADATA_SCHEMA = { + Optional("id"): int, + "filter_header": object, + "field_name": object, + "field_label": object, + "field_value": object, +} + +CUSTOM_METADATA_SCHEMA = os.getenv("CUSTOM_METADATA_SCHEMA", {}) + +EXTRA_METADATA_SCHEMA = { + **{ + "map": os.getenv("MAP_EXTRA_METADATA_SCHEMA", DEFAULT_EXTRA_METADATA_SCHEMA), + "dataset": os.getenv("DATASET_EXTRA_METADATA_SCHEMA", DEFAULT_EXTRA_METADATA_SCHEMA), + "document": os.getenv("DOCUMENT_EXTRA_METADATA_SCHEMA", DEFAULT_EXTRA_METADATA_SCHEMA), + "geoapp": os.getenv("GEOAPP_EXTRA_METADATA_SCHEMA", DEFAULT_EXTRA_METADATA_SCHEMA), + }, + **CUSTOM_METADATA_SCHEMA, +} +# ---- End deprecated ExtraMetadata settings ---- + """ List of modules that implement custom metadata storers that will be called when the metadata of a resource is saved """