Add route for importing data from UI

.github/workflows/backend-test.yaml

@@ -0,0 +1,32 @@
+name: Backend CI
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 4
+      matrix:
+        python-version: [3.12, 3.13, 3.14 ]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r ./backend/requirements.txt
+    - name: Run Tests'
+      run: |
+        cd backend
+        python manage.py migrate
+        python manage.py test

backend/Makefile

@@ -29,3 +29,6 @@ catalog:
 	coordo load file --action add data/seed/ident_sites.zip --package catalog/seed
 	coordo load file --action add data/seed/reboisement_id.zip --package catalog/seed
 	coordo load file --action add data/seed/sensib_communautaire.zip --package catalog/seed
+
+test:
+	python manage.py test

backend/maps/tests.py

@@ -1,2 +1,127 @@
+from django.contrib.auth.models import Permission
+from django.contrib.auth import get_user_model
+from django.test import TestCase, Client
+from django.core.files.uploadedfile import SimpleUploadedFile
+from django.urls import reverse
+import shutil 
 
-# Create your tests here.
+TEST_DIR = 'catalog/test'
+class FileUploadTest(TestCase):
+    def setUp(self):
+        self.client = Client()
+        self.url = reverse('maps-add-data')
+
+    def tearDown(self):
+        try:
+            shutil.rmtree(TEST_DIR)
+        except OSError:
+            pass 
+
+    def get_user_with_permission(self, username, password, codename):
+        user = get_user_model().objects.create_user(username=username, password=password)
+        permission = Permission.objects.get(codename=codename)
+        user.user_permissions.add(permission)
+        return user
+
+
+    def test_file_upload_add_ascii_content(self):
+        user = self.get_user_with_permission("testuser", "pass", "add_data")
+        self.client.force_login(user)
+
+        file_content = b'col_1,col_2\nvalue1,value2'
+
+        uploaded_file = SimpleUploadedFile(
+            name="ascii.csv",
+            content=file_content,
+            content_type='multipart/form-data'
+        )
+
+        response = self.client.post(
+            self.url,
+            data={ 
+                'file': uploaded_file, 
+                'package': TEST_DIR,
+                'action': 'add'
+            }
+        )
+
+        self.assertEqual(response.status_code, 200, f"response is {response}")
+        self.assertEqual(uploaded_file.name, response.json()['filename'])
+        self.assertIn('File uploaded successfully', response.json()['message'])
+
+
+    def test_file_upload_add_ut8_content(self):
+        user = self.get_user_with_permission("testuser", "pass", "add_data")
+        self.client.force_login(user)
+
+        file_content = 'col_1,col_2\néàë,-°$'
+
+        uploaded_file = SimpleUploadedFile(
+            name="utf8.csv",
+            content=file_content.encode('utf-8'),
+            content_type='multipart/form-data'
+        )
+
+        response = self.client.post(
+            self.url,
+            data={ 
+                'file': uploaded_file, 
+                'package': TEST_DIR,
+                'action': 'add'
+            }
+        )
+
+        self.assertEqual(response.status_code, 200, f"response is {response}")
+        self.assertEqual(uploaded_file.name, response.json()['filename'])
+        self.assertIn('File uploaded successfully', response.json()['message'])
+
+    def test_file_upload_weird_byte(self):
+        user = self.get_user_with_permission("testuser", "pass", "add_data")
+        self.client.force_login(user)
+        data = b"id,name\n1,John\n2,Ana\n3,Bob\x96\n"  # 0x96 is typical Windows-1252 dash
+
+        uploaded_file = SimpleUploadedFile(
+            name="ascii_with_weird_byte.csv",
+            content=data,
+            content_type='multipart/form-data'
+        )
+
+        response = self.client.post(
+            self.url,
+            data={ 
+                'file': uploaded_file, 
+                'package': TEST_DIR,
+                'action': 'add'
+            }
+            )
+
+        self.assertEqual(response.status_code, 200, f"response is {response}")
+        self.assertEqual(uploaded_file.name, response.json()['filename'])
+        self.assertIn('File uploaded successfully', response.json()['message'])
+
+    def test_file_mixed_encodings(self):
+        user = self.get_user_with_permission("testuser", "pass", "add_data")
+        self.client.force_login(user)
+        part_utf8 = "id,name,city\n1,Élodie,Paris\n".encode("utf-8")
+        part_latin1 = "2,José,Lisboa\n3,François,Lyon\n".encode("latin-1")
+        file_content = part_utf8 + part_latin1
+
+        uploaded_file = SimpleUploadedFile(
+            name="mixed_encoding_file.csv",
+            content=file_content,
+            content_type='multipart/form-data'
+        )
+
+        response = self.client.post(
+            self.url,
+            data={ 
+                'file': uploaded_file, 
+                'package': TEST_DIR,
+                'action': 'add'
+            }
+            )
+
+        self.assertEqual(response.status_code, 200, f"response is {response}")
+        self.assertEqual(uploaded_file.name, response.json()['filename'])
+        self.assertIn('File uploaded successfully', response.json()['message'])
+

backend/maps/urls.py

@@ -3,6 +3,7 @@
 from . import views
 
 urlpatterns = [
+    path("add-data/", views.add_data_view, name="maps-add-data"),
     re_path(r"^(?!dashboard)(?P<subpath>\w+)", views.my_map_view, name="maps-data"),
     path("dashboard/<layer_id>", views.dashboard_view, name="dashboard-data"),
 ]

backend/maps/views.py

@@ -1,16 +1,21 @@
-from django.http import HttpResponseBadRequest
+from pathlib import Path
+from tempfile import gettempdir
 
+from coordo.loaders import FileLoader, ResourceAction
 from coordo.map import Map
 from django.conf import settings
-from django.http import JsonResponse
+from django.http import JsonResponse, HttpResponseBadRequest
+from django.views.decorators.http import require_POST
 from django.views.decorators.csrf import csrf_exempt
+from django.contrib.auth.decorators import permission_required
 import pandas as pd
 import numpy as np
+import chardet
 
 config_path = settings.BASE_DIR / "configs" / "config.json"
 map = Map.from_file(config_path)
 
-@csrf_exempt
+@csrf_exempt    
 def my_map_view(request, subpath):
     return JsonResponse(
         map.handle_request(
@@ -58,3 +63,60 @@ def dashboard_view(request, layer_id):
 
     return HttpResponseBadRequest(f'Layer "{layer_id}" not yet supported', status=501)
 
+@require_POST
+@permission_required("users.add_data")
+def add_data_view(request):
+    """
+    View for adding a file to a DataPackage.
+    Expects a POST request with a body containing the 'file', 'package' and 'action' fields.
+    The uploaded file is saved temporarily in the system's temporary folder, processed and removed at the end.
+    """
+    if 'file' not in request.FILES:
+        return JsonResponse({'error': 'No file provided'}, status=400)
+
+    uploaded_file = request.FILES['file']
+    temp_file = Path(gettempdir()) / uploaded_file.name
+
+    # get file content
+    file_content = uploaded_file.read()
+
+    # detect the encoding using chardet, decode the content and re-encode as UTF-8
+    encoding_info = chardet.detect(file_content)
+    detected_encoding = encoding_info['encoding']
+    decoded_content = file_content.decode(detected_encoding)
+    utf8_content = decoded_content.encode('utf-8')
+
+    try:
+        # save the file temporarily
+        with open(temp_file, 'wb') as f:
+            f.write(utf8_content)
+
+        try:
+            package = Path(request.POST["package"])
+            action = request.POST["action"]
+        except KeyError:
+            return JsonResponse(
+                {"error": "Invalid request format. The request must contain the 'package' and 'action' fields."}, 
+                status=400
+            )
+
+        if action not in ResourceAction:
+            return JsonResponse(
+                {"error": "Invalid action. The action must be one of: " + ", ".join(ResourceAction)}, 
+                status=400
+            )
+
+        try:
+            FileLoader(package, temp_file, action).etl()
+        except Exception as e:
+            return JsonResponse({'error': str(e)}, status=500)
+
+    finally:
+        # in any case, delete the temporary file
+        temp_file.unlink(missing_ok=True)
+
+    return JsonResponse({
+            'message': 'File uploaded successfully',
+            'filename': uploaded_file.name
+        })
+

backend/requirements.txt

@@ -5,4 +5,5 @@ djangorestframework==3.16.0
 djangorestframework_simplejwt==5.5.1
 django-cors-headers==4.9.0
 docutils==0.22
-whitenoise[brotli]==6.12.0
+whitenoise[brotli]==6.12.0
+chardet==7.4.3