Write binary to stdout buffer

.github/workflows/ci.yml

@@ -10,6 +10,7 @@ jobs:
   test:
     runs-on: ubuntu-latest
     strategy:
+      max-parallel: 1  # Run tests sequentially to avoid conflicts on shared mica-demo.obiba.org server
       matrix:
         python-version: [3.8.18, 3.10.18, 3.12.11]
     steps:

obiba_mica/console.py

@@ -113,7 +113,10 @@ def run():
 
             args.func(args)
         except HTTPError as e:
-            print(e.error['status'] if e.error is not None else e)
+            if e.error is not None:
+                print(e.error.get('status', e.error))
+            else:
+                print(e)
             sys.exit(2)
     else:
       print('Mica command line tool.')

obiba_mica/core.py

@@ -190,6 +190,10 @@ def fail_on_error(self):
         self._fail_on_error = True
         return self
 
+    def ignore_fail_on_error(self):
+        self._fail_on_error = False
+        return self
+
     def header(self, key, value):
         """
         Adds a header to session headers used by the request

obiba_mica/file.py

@@ -4,12 +4,13 @@
 
 import argparse
 import json
-from obiba_mica.core import MicaClient
+from obiba_mica.core import MicaClient, HTTPError
 import urllib.request
 import urllib.parse
 import urllib.error
 import re
 import os
+import time
 
 class FileAction(argparse.Action):
   """
@@ -83,11 +84,25 @@ def __make_request(self):
   def __validate_status(self, file, status):
     """
     Validates the input request, must match the actual file status
+    Retries on 404 errors with exponential backoff to handle eventual consistency
     """
-    state = self.get(file).as_json()
-    if state['revisionStatus'] != status:
-        raise Exception('Invalid file revision status. Found: %s, Required: %s' % (
-            state['revisionStatus'], status))
+    max_retries = 7
+    retry_delay = 1
+
+    for attempt in range(max_retries):
+      try:
+        state = self.get(file).as_json()
+        if state['revisionStatus'] != status:
+          raise Exception('Invalid file revision status. Found: %s, Required: %s' % (
+              state['revisionStatus'], status))
+        return  # Success
+      except HTTPError as e:
+        if e.code == 404 and attempt < max_retries - 1:
+          # File not available yet (eventual consistency), retry with exponential backoff
+          time.sleep(retry_delay)
+          retry_delay *= 2
+        else:
+          raise  # Re-raise if not 404 or out of retries
 
   def get(self, file):
     """

obiba_mica/rest.py

@@ -103,13 +103,18 @@ def do_command(cls, args):
         response = request.send()
 
         # format response
-        res = response.as_json()
-        if args.json:
-            res = response.pretty_json()
-        elif args.method in ['OPTIONS']:
-            res = response.headers['Allow']
-
-        # output to stdout
-        print(res)
+        if args.method in ['OPTIONS']:
+            # OPTIONS method - extract Allow header
+            print(response.headers['Allow'])
+        elif 'json' not in response.headers.get('Content-Type', 'application/json').lower():
+            # Binary or non-JSON response - output raw content
+            if response.content:
+                sys.stdout.buffer.write(response.content)
+        else:
+            # JSON response - format and print
+            res = response.as_json()
+            if args.json:
+                res = response.pretty_json()
+            print(res)
     finally:
         client.close()

tests/test_access.py

@@ -4,18 +4,37 @@
 
 class TestClass(unittest.TestCase):
 
+  def setUp(self):
+    """Clean up before each test to ensure test isolation"""
+    # Clean up document access
+    try:
+      service = IndividualStudyAccessService(Utils.make_client())
+      service.delete_access('clsa', 'USER', 'user1')
+    except Exception:
+      pass
+
+    # Clean up file access
+    try:
+      service = FileAccessService(Utils.make_client())
+      file = '/individual-study/cls/population/1/data-collection-event/4/Wave 4 subject interview.pdf'
+      service.delete_access(file, 'USER', 'user1')
+    except Exception:
+      pass
+
   def test_documentAccess(self):
     self.service = IndividualStudyAccessService(Utils.make_client())
 
     try:
       response = self.service.add_access('clsa', 'USER', 'user1')
       assert response.code == 204
 
-      response = self.service.list_accesses('clsa').as_json()
-      found = next((x for x in response if x['principal'] == 'user1'), None)
+      # Wait for access to be indexed/available
+      def check_access():
+        response = self.service.list_accesses('clsa').as_json()
+        found = next((x for x in response if x['principal'] == 'user1'), None)
+        return found is not None
 
-      if found is None:
-        assert False
+      assert Utils.wait_for_condition(check_access, timeout=Utils.get_timeout(10)), "Access not found after add"
 
       response = self.service.delete_access('clsa', 'USER', 'user1')
       assert response.code == 204
@@ -30,11 +49,13 @@ def test_fileAccess(self):
       response = self.service.add_access(file, 'USER', 'user1')
       assert response.code == 204
 
-      response = self.service.list_accesses(file).as_json()
-      found = next((x for x in response if x['principal'] == 'user1'), None)
+      # Wait for access to be indexed/available
+      def check_access():
+        response = self.service.list_accesses(file).as_json()
+        found = next((x for x in response if x['principal'] == 'user1'), None)
+        return found is not None
 
-      if found is None:
-        assert False
+      assert Utils.wait_for_condition(check_access, timeout=Utils.get_timeout(10)), "File access not found after add"
 
       response = self.service.delete_access(file, 'USER', 'user1')
       assert response.code == 204

tests/test_file.py

@@ -8,12 +8,44 @@ class TestClass(unittest.TestCase):
   @classmethod
   def setup_class(cls):
     cls.service = FileService(Utils.make_client())
+    # Clean up any leftover file from previous test runs
+    cls._cleanup_test_file()
+
+  @classmethod
+  def teardown_class(cls):
+    # Clean up after all tests complete
+    cls._cleanup_test_file()
+
+  @classmethod
+  def _cleanup_test_file(cls):
+    """Clean up test file to ensure test isolation"""
+    from obiba_mica.core import HTTPError
+    try:
+      existing = cls.service.get('/individual-study/dummy.csv')
+      if existing:
+        current_status = existing.as_json().get('revisionStatus')
+        if current_status != FileService.STATUS_DELETED:
+          try:
+            cls.service.status('/individual-study/dummy.csv', FileService.STATUS_DELETED)
+          except Exception:
+            pass
+        try:
+          cls.service.delete('/individual-study/dummy.csv')
+        except Exception:
+          pass
+    except HTTPError:
+      pass  # File doesn't exist, which is fine
 
   def test_1_fileUpload(self):
     try:
       response = self.service.upload('/individual-study',  './tests/resources/dummy.csv')
 
       if response.code == 201:
+        # Wait for file to be indexed/available after upload
+        Utils.wait_for_condition(
+          lambda: self.service.get('/individual-study/dummy.csv') is not None,
+          timeout=Utils.get_timeout(10)
+        )
         assert True
       else:
         assert False
@@ -22,16 +54,22 @@ def test_1_fileUpload(self):
       assert False
 
   def __test_fileChangeStatus(self, file, status):
-    try:
-      response = self.service.status(file, status)
-
-      if response.code == 204:
-        assert True
-      else:
-        assert False
-
-    except Exception as e:
-      assert False
+    from obiba_mica.core import HTTPError
+
+    def try_status_change():
+      try:
+        response = self.service.status(file, status)
+        return response.code == 204
+      except HTTPError as e:
+        # Retry on 404 (file not indexed yet) or 5xx (server errors)
+        if e.code == 404 or e.is_server_error():
+          return False
+        raise
+
+    # Retry with exponential backoff - longer timeout in CI
+    timeout = Utils.get_timeout(7)  # 7s local, 21s in CI
+    success = Utils.wait_for_condition(try_status_change, timeout=timeout, interval=1, backoff='exponential')
+    assert success, f"Failed to change status to {status} for {file}"
 
   def __test_fileDelete(self, path):
     try:
@@ -53,6 +91,11 @@ def test_3_filePublish(self):
       response = self.service.publish('/individual-study/dummy.csv', True)
 
       if response.code == 204:
+        # Wait for publish to complete/propagate
+        Utils.wait_for_condition(
+          lambda: self.service.get('/individual-study/dummy.csv') is not None,
+          timeout=Utils.get_timeout(10)
+        )
         assert True
       else:
         assert False
@@ -89,6 +132,17 @@ def test_5_fileDownload(self):
   def test_6_changeDeletedStatus(self):
     self.__test_fileChangeStatus('/individual-study/dummy.csv', FileService.STATUS_DELETED)
 
+    # Wait for status change to propagate before test_7 tries to delete
+    def check_status():
+      try:
+        state = self.service.get('/individual-study/dummy.csv').as_json()
+        return state.get('revisionStatus') == FileService.STATUS_DELETED
+      except Exception:
+        return False
+
+    assert Utils.wait_for_condition(check_status, timeout=Utils.get_timeout(10)), \
+        "File status did not propagate to DELETED"
+
   def test_7_fileDelete(self):
     self.__test_fileDelete('/individual-study/dummy.csv')
 
@@ -98,6 +152,18 @@ def test_8_createFolder(self):
 
       if response.code == 201:
         self.__test_fileChangeStatus('/individual-study/yoyo', FileService.STATUS_DELETED)
+
+        # Wait for status change to propagate before delete
+        def check_status():
+          try:
+            state = self.service.get('/individual-study/yoyo').as_json()
+            return state.get('revisionStatus') == FileService.STATUS_DELETED
+          except Exception:
+            return False
+
+        assert Utils.wait_for_condition(check_status, timeout=Utils.get_timeout(10)), \
+            "Folder status did not propagate to DELETED"
+
         self.__test_fileDelete('/individual-study/yoyo')
 
       else: