From c0b4593e716bd51d0cb617dfb73a597244d42533 Mon Sep 17 00:00:00 2001 From: Olivier Date: Thu, 30 Apr 2026 14:01:02 +0200 Subject: [PATCH 01/11] add view --- backend/maps/urls.py | 1 + backend/maps/views.py | 46 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/backend/maps/urls.py b/backend/maps/urls.py index ab7f561..0716d12 100644 --- a/backend/maps/urls.py +++ b/backend/maps/urls.py @@ -3,6 +3,7 @@ from . import views urlpatterns = [ + path("add-data/", views.add_data_view, name="maps-add-data"), re_path(r"^(?!dashboard)(?P\w+)", views.my_map_view, name="maps-data"), path("dashboard/", views.dashboard_view, name="dashboard-data"), ] diff --git a/backend/maps/views.py b/backend/maps/views.py index f103e80..b2e1b67 100644 --- a/backend/maps/views.py +++ b/backend/maps/views.py @@ -1,9 +1,14 @@ from django.http import HttpResponseBadRequest +from pathlib import Path +from tempfile import NamedTemporaryFile +from coordo.loaders import FileLoader, ResourceAction from coordo.map import Map from django.conf import settings from django.http import JsonResponse +from django.views.decorators.http import require_POST from django.views.decorators.csrf import csrf_exempt +from django.contrib.auth.decorators import permission_required import pandas as pd import numpy as np @@ -58,3 +63,44 @@ def dashboard_view(request, layer_id): return HttpResponseBadRequest(f'Layer "{layer_id}" not yet supported', status=501) +@require_POST +@permission_required("users.add_data") +def add_data_view(request): + if 'file' not in request.FILES: + return JsonResponse({'error': 'No file provided'}, status=400) + + uploaded_file = request.FILES['file'] + uploaded_file_suffix = Path(uploaded_file.name).suffix + + # NOTE: we do not use FileSystemStorage here, as it creates file name contanining both lower and upper case letters + # but pydantic does not accept upper case letters in file names + # furthermore, a temp file is more appropriate here because it is deleted automatically after context manager exits + with NamedTemporaryFile(suffix=uploaded_file_suffix) as temp_file: + temp_file.write(uploaded_file.read()) + file = Path(temp_file.name) + + try: + package = Path(request.POST["package"]) + action = request.POST["action"] + except KeyError: + return JsonResponse( + {"error": "Invalid request format. The request must contain the 'package' and 'action' fields."}, + status=400 + ) + + if action not in ResourceAction: + return JsonResponse( + {"error": "Invalid action. The action must be one of: " + ", ".join(ResourceAction)}, + status=400 + ) + + try: + FileLoader(package, file, action).etl() + except Exception as e: + return JsonResponse({'error': str(e)}, status=500) + + return JsonResponse({ + 'message': 'File uploaded successfully', + 'filename': uploaded_file.name + }) + \ No newline at end of file From c638bdb95f595edcf05561380cbaf81bf2f96e09 Mon Sep 17 00:00:00 2001 From: Olivier Date: Thu, 30 Apr 2026 14:01:09 +0200 Subject: [PATCH 02/11] add tests --- backend/Makefile | 3 +++ backend/maps/tests.py | 39 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/backend/Makefile b/backend/Makefile index 53d0a43..f7c19cc 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -29,3 +29,6 @@ catalog: coordo load file --action add data/seed/ident_sites.zip --package catalog/seed coordo load file --action add data/seed/reboisement_id.zip --package catalog/seed coordo load file --action add data/seed/sensib_communautaire.zip --package catalog/seed + +test: + python manage.py test \ No newline at end of file diff --git a/backend/maps/tests.py b/backend/maps/tests.py index 4929020..f54b298 100644 --- a/backend/maps/tests.py +++ b/backend/maps/tests.py @@ -1,2 +1,39 @@ +from django.contrib.auth.models import Permission +from django.contrib.auth import get_user_model +from django.test import TestCase, Client +from django.core.files.uploadedfile import SimpleUploadedFile +from django.urls import reverse -# Create your tests here. + +class FileUploadTest(TestCase): + def setUp(self): + self.client = Client() + self.url = reverse('maps-add-data') + + def test_file_upload_add(self): + user = get_user_model().objects.create_user(username="testuser", password="pass") + permission = Permission.objects.get(codename="add_data") + user.user_permissions.add(permission) + + self.client.force_login(user) + + file_content = b'col_1,col_2\nvalue1,value2' + + uploaded_file = SimpleUploadedFile( + name="test.csv", + content=file_content, + content_type='multipart/form-data' + ) + + response = self.client.post( + self.url, + data={ + 'file': uploaded_file, + 'package': 'catalog/test', + 'action': 'add' + } + ) + + self.assertEqual(response.status_code, 200) + self.assertIn('File uploaded successfully', response.json()['message']) + \ No newline at end of file From 0e0729351ea3ada40683c9a310934ac897e2db72 Mon Sep 17 00:00:00 2001 From: Olivier Date: Mon, 4 May 2026 19:20:58 +0200 Subject: [PATCH 03/11] fix imports --- backend/maps/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/maps/views.py b/backend/maps/views.py index b2e1b67..d57e43c 100644 --- a/backend/maps/views.py +++ b/backend/maps/views.py @@ -1,11 +1,10 @@ -from django.http import HttpResponseBadRequest from pathlib import Path from tempfile import NamedTemporaryFile from coordo.loaders import FileLoader, ResourceAction from coordo.map import Map from django.conf import settings -from django.http import JsonResponse +from django.http import JsonResponse, HttpResponseBadRequest from django.views.decorators.http import require_POST from django.views.decorators.csrf import csrf_exempt from django.contrib.auth.decorators import permission_required @@ -72,6 +71,7 @@ def add_data_view(request): uploaded_file = request.FILES['file'] uploaded_file_suffix = Path(uploaded_file.name).suffix + # storing temporarily the uploaded file # NOTE: we do not use FileSystemStorage here, as it creates file name contanining both lower and upper case letters # but pydantic does not accept upper case letters in file names # furthermore, a temp file is more appropriate here because it is deleted automatically after context manager exits From 1d0991b153a555c74b61843d5ad5dac7b027549e Mon Sep 17 00:00:00 2001 From: Olivier Date: Mon, 4 May 2026 19:41:25 +0200 Subject: [PATCH 04/11] fix issue with woring filename --- backend/maps/tests.py | 1 + backend/maps/views.py | 31 +++++++++++++++++++------------ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/backend/maps/tests.py b/backend/maps/tests.py index f54b298..9602377 100644 --- a/backend/maps/tests.py +++ b/backend/maps/tests.py @@ -35,5 +35,6 @@ def test_file_upload_add(self): ) self.assertEqual(response.status_code, 200) + self.assertEqual(uploaded_file.name, response.json()['filename']) self.assertIn('File uploaded successfully', response.json()['message']) \ No newline at end of file diff --git a/backend/maps/views.py b/backend/maps/views.py index d57e43c..3e0cf12 100644 --- a/backend/maps/views.py +++ b/backend/maps/views.py @@ -1,5 +1,5 @@ from pathlib import Path -from tempfile import NamedTemporaryFile +from tempfile import gettempdir from coordo.loaders import FileLoader, ResourceAction from coordo.map import Map @@ -14,7 +14,7 @@ config_path = settings.BASE_DIR / "configs" / "config.json" map = Map.from_file(config_path) -@csrf_exempt +@csrf_exempt def my_map_view(request, subpath): return JsonResponse( map.handle_request( @@ -65,19 +65,22 @@ def dashboard_view(request, layer_id): @require_POST @permission_required("users.add_data") def add_data_view(request): + """ + View for adding a file to a DataPackage. + Expects a POST request with a body containing the 'file', 'package' and 'action' fields. + The uploaded file is saved temporarily in the system's temporary folder, processed and removed at the end. + """ if 'file' not in request.FILES: return JsonResponse({'error': 'No file provided'}, status=400) uploaded_file = request.FILES['file'] - uploaded_file_suffix = Path(uploaded_file.name).suffix - - # storing temporarily the uploaded file - # NOTE: we do not use FileSystemStorage here, as it creates file name contanining both lower and upper case letters - # but pydantic does not accept upper case letters in file names - # furthermore, a temp file is more appropriate here because it is deleted automatically after context manager exits - with NamedTemporaryFile(suffix=uploaded_file_suffix) as temp_file: - temp_file.write(uploaded_file.read()) - file = Path(temp_file.name) + temp_file = Path(gettempdir()) / uploaded_file.name + + try: + # save the file temporarily + with open(temp_file, 'wb+') as f: + for chunk in uploaded_file.chunks(): + f.write(chunk) try: package = Path(request.POST["package"]) @@ -95,9 +98,13 @@ def add_data_view(request): ) try: - FileLoader(package, file, action).etl() + FileLoader(package, temp_file, action).etl() except Exception as e: return JsonResponse({'error': str(e)}, status=500) + + finally: + # in any case, delete the temporary file + temp_file.unlink(missing_ok=True) return JsonResponse({ 'message': 'File uploaded successfully', From d27644977cf4d7879a12ba50cf53afba2b04e0f6 Mon Sep 17 00:00:00 2001 From: Olivier Date: Mon, 4 May 2026 20:03:18 +0200 Subject: [PATCH 05/11] add charset detection and automatic encoding in utf8 --- backend/maps/tests.py | 38 ++++++++++++++++++++++++++++++++++---- backend/maps/views.py | 15 ++++++++++++--- backend/requirements.txt | 3 ++- 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/backend/maps/tests.py b/backend/maps/tests.py index 9602377..c9d4c37 100644 --- a/backend/maps/tests.py +++ b/backend/maps/tests.py @@ -10,17 +10,21 @@ def setUp(self): self.client = Client() self.url = reverse('maps-add-data') - def test_file_upload_add(self): - user = get_user_model().objects.create_user(username="testuser", password="pass") - permission = Permission.objects.get(codename="add_data") + def get_user_with_permission(self, username, password, codename): + user = get_user_model().objects.create_user(username=username, password=password) + permission = Permission.objects.get(codename=codename) user.user_permissions.add(permission) + return user + + def test_file_upload_add_ascii_content(self): + user = self.get_user_with_permission("testuser", "pass", "add_data") self.client.force_login(user) file_content = b'col_1,col_2\nvalue1,value2' uploaded_file = SimpleUploadedFile( - name="test.csv", + name="ascii.csv", content=file_content, content_type='multipart/form-data' ) @@ -37,4 +41,30 @@ def test_file_upload_add(self): self.assertEqual(response.status_code, 200) self.assertEqual(uploaded_file.name, response.json()['filename']) self.assertIn('File uploaded successfully', response.json()['message']) + + + def test_file_upload_add_ut8_content(self): + user = self.get_user_with_permission("testuser", "pass", "add_data") + self.client.force_login(user) + + file_content = 'col_1,col_2\néàë,-°$' + + uploaded_file = SimpleUploadedFile( + name="utf8.csv", + content=file_content.encode('utf-8'), + content_type='multipart/form-data' + ) + + response = self.client.post( + self.url, + data={ + 'file': uploaded_file, + 'package': 'catalog/test', + 'action': 'add' + } + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual(uploaded_file.name, response.json()['filename']) + self.assertIn('File uploaded successfully', response.json()['message']) \ No newline at end of file diff --git a/backend/maps/views.py b/backend/maps/views.py index 3e0cf12..9f2f08d 100644 --- a/backend/maps/views.py +++ b/backend/maps/views.py @@ -10,6 +10,7 @@ from django.contrib.auth.decorators import permission_required import pandas as pd import numpy as np +import chardet config_path = settings.BASE_DIR / "configs" / "config.json" map = Map.from_file(config_path) @@ -76,11 +77,19 @@ def add_data_view(request): uploaded_file = request.FILES['file'] temp_file = Path(gettempdir()) / uploaded_file.name + # get file content + file_content = uploaded_file.read() + + # detect the encoding using chardet, decode the content and re-encode as UTF-8 + encoding_info = chardet.detect(file_content) + detected_encoding = encoding_info['encoding'] + decoded_content = file_content.decode(detected_encoding) + utf8_content = decoded_content.encode('utf-8') + try: # save the file temporarily - with open(temp_file, 'wb+') as f: - for chunk in uploaded_file.chunks(): - f.write(chunk) + with open(temp_file, 'wb') as f: + f.write(utf8_content) try: package = Path(request.POST["package"]) diff --git a/backend/requirements.txt b/backend/requirements.txt index 966113b..b9a607d 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -5,4 +5,5 @@ djangorestframework==3.16.0 djangorestframework_simplejwt==5.5.1 django-cors-headers==4.9.0 docutils==0.22 -whitenoise[brotli]==6.12.0 \ No newline at end of file +whitenoise[brotli]==6.12.0 +chardet==7.4.3 \ No newline at end of file From 1d0a38493fbedda823213b085f04ba00f71ff77c Mon Sep 17 00:00:00 2001 From: Arnaud Fournier Date: Tue, 5 May 2026 10:38:08 +0200 Subject: [PATCH 06/11] Enhance tests and add git worklfow --- .github/workflows/backend-test.yaml | 31 +++++++++++++ backend/maps/tests.py | 67 ++++++++++++++++++++++++++--- 2 files changed, 93 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/backend-test.yaml diff --git a/.github/workflows/backend-test.yaml b/.github/workflows/backend-test.yaml new file mode 100644 index 0000000..ac9f6cd --- /dev/null +++ b/.github/workflows/backend-test.yaml @@ -0,0 +1,31 @@ +name: Backend CI + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + test: + + runs-on: ubuntu-latest + strategy: + max-parallel: 4 + matrix: + python-version: [3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r ./backend/requirements.txt + - name: Run Tests' + run: | + python ./backend/manage.py migrate + python ./backend/manage.py test \ No newline at end of file diff --git a/backend/maps/tests.py b/backend/maps/tests.py index c9d4c37..f8c46fa 100644 --- a/backend/maps/tests.py +++ b/backend/maps/tests.py @@ -3,13 +3,20 @@ from django.test import TestCase, Client from django.core.files.uploadedfile import SimpleUploadedFile from django.urls import reverse +import shutil - +TEST_DIR = 'catalog/test' class FileUploadTest(TestCase): def setUp(self): self.client = Client() self.url = reverse('maps-add-data') + def tearDown(self): + try: + shutil.rmtree(TEST_DIR) + except OSError: + pass + def get_user_with_permission(self, username, password, codename): user = get_user_model().objects.create_user(username=username, password=password) permission = Permission.objects.get(codename=codename) @@ -33,12 +40,12 @@ def test_file_upload_add_ascii_content(self): self.url, data={ 'file': uploaded_file, - 'package': 'catalog/test', + 'package': TEST_DIR, 'action': 'add' } ) - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 200, f"response is {response}") self.assertEqual(uploaded_file.name, response.json()['filename']) self.assertIn('File uploaded successfully', response.json()['message']) @@ -59,12 +66,62 @@ def test_file_upload_add_ut8_content(self): self.url, data={ 'file': uploaded_file, - 'package': 'catalog/test', + 'package': TEST_DIR, 'action': 'add' } ) - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 200, f"response is {response}") + self.assertEqual(uploaded_file.name, response.json()['filename']) + self.assertIn('File uploaded successfully', response.json()['message']) + + def test_file_upload_weird_byte(self): + user = self.get_user_with_permission("testuser", "pass", "add_data") + self.client.force_login(user) + data = b"id,name\n1,John\n2,Ana\n3,Bob\x96\n" # 0x96 is typical Windows-1252 dash + + uploaded_file = SimpleUploadedFile( + name="ascii_with_weird_byte.csv", + content=data, + content_type='multipart/form-data' + ) + + response = self.client.post( + self.url, + data={ + 'file': uploaded_file, + 'package': TEST_DIR, + 'action': 'add' + } + ) + + self.assertEqual(response.status_code, 200, f"response is {response}") + self.assertEqual(uploaded_file.name, response.json()['filename']) + self.assertIn('File uploaded successfully', response.json()['message']) + + def test_file_mixed_encodings(self): + user = self.get_user_with_permission("testuser", "pass", "add_data") + self.client.force_login(user) + part_utf8 = "id,name,city\n1,Élodie,Paris\n".encode("utf-8") + part_latin1 = "2,José,Lisboa\n3,François,Lyon\n".encode("latin-1") + file_content = part_utf8 + part_latin1 + + uploaded_file = SimpleUploadedFile( + name="mixed_encoding_file.csv", + content=file_content, + content_type='multipart/form-data' + ) + + response = self.client.post( + self.url, + data={ + 'file': uploaded_file, + 'package': TEST_DIR, + 'action': 'add' + } + ) + + self.assertEqual(response.status_code, 200, f"response is {response}") self.assertEqual(uploaded_file.name, response.json()['filename']) self.assertIn('File uploaded successfully', response.json()['message']) \ No newline at end of file From 0a8e4036a09ba4bf91af8245d9031faa4dc0736b Mon Sep 17 00:00:00 2001 From: Arnaud Fournier Date: Tue, 5 May 2026 10:39:58 +0200 Subject: [PATCH 07/11] Fix python versions --- .github/workflows/backend-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backend-test.yaml b/.github/workflows/backend-test.yaml index ac9f6cd..0c4e5fd 100644 --- a/.github/workflows/backend-test.yaml +++ b/.github/workflows/backend-test.yaml @@ -13,7 +13,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: [3.6, 3.7, 3.8] + python-version: ['3.6.7', '3.7.1', '3.8.0'] steps: - uses: actions/checkout@v2 From ef72ad2b4ea9ca1c3f9eb6e2af0dc82df66770a6 Mon Sep 17 00:00:00 2001 From: Arnaud Fournier Date: Tue, 5 May 2026 10:42:29 +0200 Subject: [PATCH 08/11] Use correct python versions --- .github/workflows/backend-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backend-test.yaml b/.github/workflows/backend-test.yaml index 0c4e5fd..e2cf156 100644 --- a/.github/workflows/backend-test.yaml +++ b/.github/workflows/backend-test.yaml @@ -13,7 +13,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: ['3.6.7', '3.7.1', '3.8.0'] + python-version: [3.11, 3.12, 3.13, 3.14 ] steps: - uses: actions/checkout@v2 From 48d7c86e554e0652bff8787c655a2d9140b2656c Mon Sep 17 00:00:00 2001 From: Arnaud Fournier Date: Tue, 5 May 2026 10:44:40 +0200 Subject: [PATCH 09/11] Change command to actually run test --- .github/workflows/backend-test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backend-test.yaml b/.github/workflows/backend-test.yaml index e2cf156..fafe7e6 100644 --- a/.github/workflows/backend-test.yaml +++ b/.github/workflows/backend-test.yaml @@ -27,5 +27,5 @@ jobs: pip install -r ./backend/requirements.txt - name: Run Tests' run: | - python ./backend/manage.py migrate - python ./backend/manage.py test \ No newline at end of file + cd backend && python manage.py migrate + cd backend && python manage.py test \ No newline at end of file From 3dc8453f4975fe11ddc2d6a5c165bf0dc1b09617 Mon Sep 17 00:00:00 2001 From: Arnaud Fournier Date: Tue, 5 May 2026 10:46:18 +0200 Subject: [PATCH 10/11] Fix syntax --- .github/workflows/backend-test.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backend-test.yaml b/.github/workflows/backend-test.yaml index fafe7e6..90f3fe4 100644 --- a/.github/workflows/backend-test.yaml +++ b/.github/workflows/backend-test.yaml @@ -27,5 +27,6 @@ jobs: pip install -r ./backend/requirements.txt - name: Run Tests' run: | - cd backend && python manage.py migrate - cd backend && python manage.py test \ No newline at end of file + cd backend + python manage.py migrate + python manage.py test \ No newline at end of file From 35c14a2b114011e981c191b9c91e6f05ab5be5e7 Mon Sep 17 00:00:00 2001 From: Arnaud Fournier Date: Tue, 5 May 2026 11:18:12 +0200 Subject: [PATCH 11/11] Remove python 3.11 --- .github/workflows/backend-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backend-test.yaml b/.github/workflows/backend-test.yaml index 90f3fe4..e7c20c1 100644 --- a/.github/workflows/backend-test.yaml +++ b/.github/workflows/backend-test.yaml @@ -13,7 +13,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: [3.11, 3.12, 3.13, 3.14 ] + python-version: [3.12, 3.13, 3.14 ] steps: - uses: actions/checkout@v2