From c8f94f1f015142331c933be2eb352e24d9e55a62 Mon Sep 17 00:00:00 2001
From: Ivan Ivanov <ivan.ivanov@suricactus.com>
Date: Mon, 20 Oct 2025 17:47:38 +0300
Subject: [PATCH] feat: add multipart encoder when uploading (large) files

With the previous implementation of `upload_files`, the SDK was loading
the whole file to be uploaded in the memory and only then sending it
over the socket. This was painfully slow, memory constrained and lacked
real feedback.

Therefore the `requests_toolbelt`'s `MultipartEncoderMonitor` is
introduced in this PR, which keeps the memory consumption under control.

This is quite useful not only for CLI consumers of the SDK, but also for
QFieldCloud `qgis` workers, as the memory does not grow too much when
uploading large files after `package` or `apply_deltas` job.
---
 qfieldcloud_sdk/sdk.py | 35 ++++++++++++++++++++++++++++-------
 requirements.txt       |  1 +
 2 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/qfieldcloud_sdk/sdk.py b/qfieldcloud_sdk/sdk.py
index a968876..27d8f35 100644
--- a/qfieldcloud_sdk/sdk.py
+++ b/qfieldcloud_sdk/sdk.py
@@ -11,6 +11,7 @@
 import requests
 import urllib3
 from requests.adapters import HTTPAdapter, Retry
+from requests_toolbelt.multipart.encoder import MultipartEncoderMonitor
 
 from .interfaces import QfcException, QfcRequest, QfcRequestException
 from .utils import calc_etag, log, add_trailing_slash_to_url
@@ -665,21 +666,39 @@ def upload_file(
         # if the filepath is invalid, it will throw a new error `pathvalidate.ValidationError`
         is_valid_filepath(str(local_filename))
 
+        local_file_size = local_filename.stat().st_size
         with open(local_filename, "rb") as local_file:
-            upload_file = local_file
+            encoder_params = {}
+
             if show_progress:
                 from tqdm import tqdm
-                from tqdm.utils import CallbackIOWrapper
 
                 progress_bar = tqdm(
-                    total=local_filename.stat().st_size,
+                    total=local_file_size,
                     unit_scale=True,
-                    desc=local_filename.stem,
+                    unit="B",
+                    desc=f'Uploading "{remote_filename}"...',
                 )
-                upload_file = CallbackIOWrapper(progress_bar.update, local_file, "read")
+
+                def cb(monitor: MultipartEncoderMonitor) -> None:
+                    progress_bar.n = monitor.bytes_read
+                    progress_bar.refresh()
+
+                encoder_params["callback"] = cb
             else:
                 logger.info(f'Uploading file "{remote_filename}"…')
 
+            multipart_data = MultipartEncoderMonitor.from_fields(
+                fields={
+                    "file": (
+                        str(remote_filename),
+                        local_file,
+                        None,
+                    ),
+                },
+                **encoder_params,
+            )
+
             if upload_type == FileTransferType.PROJECT:
                 url = f"files/{project_id}/{remote_filename}"
             elif upload_type == FileTransferType.PACKAGE:
@@ -695,8 +714,10 @@ def upload_file(
             return self._request(
                 "POST",
                 url,
-                files={
-                    "file": upload_file,
+                data=multipart_data,
+                headers={
+                    "Content-Type": multipart_data.content_type,
+                    "Accept": "application/json",
                 },
             )
 
diff --git a/requirements.txt b/requirements.txt
index 7f57a4c..da3f6a7 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,6 +3,7 @@ charset-normalizer>=3.2.0
 click>=8.1.5
 idna>=3.4
 requests>=2.31.0
+requests-toolbelt>=1.0.0
 tqdm>=4.65.0
 urllib3>=2.0.7
 pathvalidate>=3.2.1