diff --git a/.github/workflows/backend-test.yaml b/.github/workflows/backend-test.yaml new file mode 100644 index 00000000..e7c20c16 --- /dev/null +++ b/.github/workflows/backend-test.yaml @@ -0,0 +1,32 @@ +name: Backend CI + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + test: + + runs-on: ubuntu-latest + strategy: + max-parallel: 4 + matrix: + python-version: [3.12, 3.13, 3.14 ] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r ./backend/requirements.txt + - name: Run Tests' + run: | + cd backend + python manage.py migrate + python manage.py test \ No newline at end of file diff --git a/backend/Makefile b/backend/Makefile index 53d0a435..f7c19cc7 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -29,3 +29,6 @@ catalog: coordo load file --action add data/seed/ident_sites.zip --package catalog/seed coordo load file --action add data/seed/reboisement_id.zip --package catalog/seed coordo load file --action add data/seed/sensib_communautaire.zip --package catalog/seed + +test: + python manage.py test \ No newline at end of file diff --git a/backend/maps/tests.py b/backend/maps/tests.py index 49290204..f8c46fac 100644 --- a/backend/maps/tests.py +++ b/backend/maps/tests.py @@ -1,2 +1,127 @@ +from django.contrib.auth.models import Permission +from django.contrib.auth import get_user_model +from django.test import TestCase, Client +from django.core.files.uploadedfile import SimpleUploadedFile +from django.urls import reverse +import shutil -# Create your tests here. +TEST_DIR = 'catalog/test' +class FileUploadTest(TestCase): + def setUp(self): + self.client = Client() + self.url = reverse('maps-add-data') + + def tearDown(self): + try: + shutil.rmtree(TEST_DIR) + except OSError: + pass + + def get_user_with_permission(self, username, password, codename): + user = get_user_model().objects.create_user(username=username, password=password) + permission = Permission.objects.get(codename=codename) + user.user_permissions.add(permission) + return user + + + def test_file_upload_add_ascii_content(self): + user = self.get_user_with_permission("testuser", "pass", "add_data") + self.client.force_login(user) + + file_content = b'col_1,col_2\nvalue1,value2' + + uploaded_file = SimpleUploadedFile( + name="ascii.csv", + content=file_content, + content_type='multipart/form-data' + ) + + response = self.client.post( + self.url, + data={ + 'file': uploaded_file, + 'package': TEST_DIR, + 'action': 'add' + } + ) + + self.assertEqual(response.status_code, 200, f"response is {response}") + self.assertEqual(uploaded_file.name, response.json()['filename']) + self.assertIn('File uploaded successfully', response.json()['message']) + + + def test_file_upload_add_ut8_content(self): + user = self.get_user_with_permission("testuser", "pass", "add_data") + self.client.force_login(user) + + file_content = 'col_1,col_2\néàë,-°$' + + uploaded_file = SimpleUploadedFile( + name="utf8.csv", + content=file_content.encode('utf-8'), + content_type='multipart/form-data' + ) + + response = self.client.post( + self.url, + data={ + 'file': uploaded_file, + 'package': TEST_DIR, + 'action': 'add' + } + ) + + self.assertEqual(response.status_code, 200, f"response is {response}") + self.assertEqual(uploaded_file.name, response.json()['filename']) + self.assertIn('File uploaded successfully', response.json()['message']) + + def test_file_upload_weird_byte(self): + user = self.get_user_with_permission("testuser", "pass", "add_data") + self.client.force_login(user) + data = b"id,name\n1,John\n2,Ana\n3,Bob\x96\n" # 0x96 is typical Windows-1252 dash + + uploaded_file = SimpleUploadedFile( + name="ascii_with_weird_byte.csv", + content=data, + content_type='multipart/form-data' + ) + + response = self.client.post( + self.url, + data={ + 'file': uploaded_file, + 'package': TEST_DIR, + 'action': 'add' + } + ) + + self.assertEqual(response.status_code, 200, f"response is {response}") + self.assertEqual(uploaded_file.name, response.json()['filename']) + self.assertIn('File uploaded successfully', response.json()['message']) + + def test_file_mixed_encodings(self): + user = self.get_user_with_permission("testuser", "pass", "add_data") + self.client.force_login(user) + part_utf8 = "id,name,city\n1,Élodie,Paris\n".encode("utf-8") + part_latin1 = "2,José,Lisboa\n3,François,Lyon\n".encode("latin-1") + file_content = part_utf8 + part_latin1 + + uploaded_file = SimpleUploadedFile( + name="mixed_encoding_file.csv", + content=file_content, + content_type='multipart/form-data' + ) + + response = self.client.post( + self.url, + data={ + 'file': uploaded_file, + 'package': TEST_DIR, + 'action': 'add' + } + ) + + self.assertEqual(response.status_code, 200, f"response is {response}") + self.assertEqual(uploaded_file.name, response.json()['filename']) + self.assertIn('File uploaded successfully', response.json()['message']) + \ No newline at end of file diff --git a/backend/maps/urls.py b/backend/maps/urls.py index ab7f561e..0716d125 100644 --- a/backend/maps/urls.py +++ b/backend/maps/urls.py @@ -3,6 +3,7 @@ from . import views urlpatterns = [ + path("add-data/", views.add_data_view, name="maps-add-data"), re_path(r"^(?!dashboard)(?P\w+)", views.my_map_view, name="maps-data"), path("dashboard/", views.dashboard_view, name="dashboard-data"), ] diff --git a/backend/maps/views.py b/backend/maps/views.py index f103e80b..9f2f08d0 100644 --- a/backend/maps/views.py +++ b/backend/maps/views.py @@ -1,16 +1,21 @@ -from django.http import HttpResponseBadRequest +from pathlib import Path +from tempfile import gettempdir +from coordo.loaders import FileLoader, ResourceAction from coordo.map import Map from django.conf import settings -from django.http import JsonResponse +from django.http import JsonResponse, HttpResponseBadRequest +from django.views.decorators.http import require_POST from django.views.decorators.csrf import csrf_exempt +from django.contrib.auth.decorators import permission_required import pandas as pd import numpy as np +import chardet config_path = settings.BASE_DIR / "configs" / "config.json" map = Map.from_file(config_path) -@csrf_exempt +@csrf_exempt def my_map_view(request, subpath): return JsonResponse( map.handle_request( @@ -58,3 +63,60 @@ def dashboard_view(request, layer_id): return HttpResponseBadRequest(f'Layer "{layer_id}" not yet supported', status=501) +@require_POST +@permission_required("users.add_data") +def add_data_view(request): + """ + View for adding a file to a DataPackage. + Expects a POST request with a body containing the 'file', 'package' and 'action' fields. + The uploaded file is saved temporarily in the system's temporary folder, processed and removed at the end. + """ + if 'file' not in request.FILES: + return JsonResponse({'error': 'No file provided'}, status=400) + + uploaded_file = request.FILES['file'] + temp_file = Path(gettempdir()) / uploaded_file.name + + # get file content + file_content = uploaded_file.read() + + # detect the encoding using chardet, decode the content and re-encode as UTF-8 + encoding_info = chardet.detect(file_content) + detected_encoding = encoding_info['encoding'] + decoded_content = file_content.decode(detected_encoding) + utf8_content = decoded_content.encode('utf-8') + + try: + # save the file temporarily + with open(temp_file, 'wb') as f: + f.write(utf8_content) + + try: + package = Path(request.POST["package"]) + action = request.POST["action"] + except KeyError: + return JsonResponse( + {"error": "Invalid request format. The request must contain the 'package' and 'action' fields."}, + status=400 + ) + + if action not in ResourceAction: + return JsonResponse( + {"error": "Invalid action. The action must be one of: " + ", ".join(ResourceAction)}, + status=400 + ) + + try: + FileLoader(package, temp_file, action).etl() + except Exception as e: + return JsonResponse({'error': str(e)}, status=500) + + finally: + # in any case, delete the temporary file + temp_file.unlink(missing_ok=True) + + return JsonResponse({ + 'message': 'File uploaded successfully', + 'filename': uploaded_file.name + }) + \ No newline at end of file diff --git a/backend/requirements.txt b/backend/requirements.txt index 966113ba..b9a607d4 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -5,4 +5,5 @@ djangorestframework==3.16.0 djangorestframework_simplejwt==5.5.1 django-cors-headers==4.9.0 docutils==0.22 -whitenoise[brotli]==6.12.0 \ No newline at end of file +whitenoise[brotli]==6.12.0 +chardet==7.4.3 \ No newline at end of file