Aletournel/synth 24202 improve verbosity and debug to avoid stuck impression

datadog_sync/model/synthetics_global_variables.py

@@ -37,8 +37,13 @@ class SyntheticsGlobalVariables(BaseResource):
     destination_global_variables: Dict[str, Dict] = dict()
 
     async def get_resources(self, client: CustomClient) -> List[Dict]:
+        self.config.logger.debug(
+            "synthetics_global_variables: fetching from %s", self.resource_config.base_path
+        )
         resp = await client.get(self.resource_config.base_path)
-        return resp["variables"]
+        variables = resp.get("variables", [])
+        self.config.logger.debug("synthetics_global_variables: got %d variables", len(variables))
+        return variables
 
     async def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = None) -> Tuple[str, Dict]:
         if _id:

datadog_sync/model/synthetics_mobile_applications_versions.py

@@ -39,14 +39,51 @@ async def get_resources(self, client: CustomClient) -> List[Dict]:
         """
         Mobile Application Versions don't have a list endpoint of their own
         """
+        self.config.logger.debug(
+            "synthetics_mobile_applications_versions: fetching applications list from %s",
+            self.applications_path,
+        )
         resp = await client.get(self.applications_path)
+        applications = resp.get("applications", [])
+        total_versions = sum(len(app.get("versions", [])) for app in applications)
+        self.config.logger.info(
+            "synthetics_mobile_applications_versions: %d applications, %d versions to fetch",
+            len(applications),
+            total_versions,
+        )
+
+        # If import progress bar is active, add sub-steps so the bar advances during fetch
+        add_total = getattr(self.config, "_import_progress_add_total", None)
+        if callable(add_total):
+            add_total(total_versions)
 
         resources = []
-        for application in resp["applications"]:
-            for version in application["versions"]:
+        progress_interval = max(1, total_versions // 20)  # log at INFO roughly every 5%
+        progress_update = getattr(self.config, "_import_progress_update", None)
+        for app_idx, application in enumerate(applications):
+            versions = application.get("versions", [])
+            for ver_idx, version in enumerate(versions):
                 _id = version["id"]
+                current = len(resources) + 1
+                self.config.logger.debug(
+                    "synthetics_mobile_applications_versions: fetching version %s (%d/%d)",
+                    _id,
+                    current,
+                    total_versions,
+                )
                 resource = await client.get(self.resource_config.base_path + f"/{_id}")
                 resources.append(resource)
+                if callable(progress_update):
+                    progress_update(1)
+                if current % progress_interval == 0 or current == total_versions:
+                    self.config.logger.info(
+                        "synthetics_mobile_applications_versions: progress %d/%d",
+                        current,
+                        total_versions,
+                    )
+        self.config.logger.info(
+            "synthetics_mobile_applications_versions: finished fetching %d versions", len(resources)
+        )
         return resources
 
     async def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = None) -> Tuple[str, Dict]:

datadog_sync/model/synthetics_tests.py

@@ -74,13 +74,17 @@ class SyntheticsTests(BaseResource):
     versions: List = []
 
     async def get_resources(self, client: CustomClient) -> List[Dict]:
+        self.config.logger.debug("synthetics_tests: fetching list from %s", self.resource_config.base_path)
         resp = await client.get(
             self.resource_config.base_path,
             params=self.get_params,
         )
+        tests = resp.get("tests", [])
+        self.config.logger.debug("synthetics_tests: got %d tests, fetching mobile application versions...", len(tests))
         versions = SyntheticsMobileApplicationsVersions(self.config)
         self.versions = await versions.get_resources(client)
-        return resp["tests"]
+        self.config.logger.debug("synthetics_tests: got %d mobile app versions", len(self.versions))
+        return tests
 
     async def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = None) -> Tuple[str, Dict]:
         source_client = self.config.source_client

datadog_sync/utils/workers.py

@@ -95,13 +95,30 @@ async def schedule_workers(self, additional_coros: List = []) -> Future:
     async def schedule_workers_with_pbar(self, total, additional_coros: List = []) -> Future:
         self.pbar = tqdm(total=total)
 
-        self._shutdown_workers = False
-        with logging_redirect_tqdm():
-            additional_coros.append(self._refresh_pbar())
-            await self.schedule_workers(additional_coros)
+        # Allow long-running get_resources (e.g. mobile app versions) to update the bar
+        def _update(n: int = 1) -> None:
+            if self.pbar:
+                self.pbar.update(n)
 
-        self.pbar.close()
-        self.pbar = None
+        def _add_total(n: int) -> None:
+            if self.pbar:
+                self.pbar.total += n
+                self.pbar.refresh()
+
+        setattr(self.config, "_import_progress_update", _update)
+        setattr(self.config, "_import_progress_add_total", _add_total)
+
+        self._shutdown_workers = False
+        try:
+            with logging_redirect_tqdm():
+                additional_coros.append(self._refresh_pbar())
+                await self.schedule_workers(additional_coros)
+        finally:
+            for attr in ("_import_progress_update", "_import_progress_add_total"):
+                if hasattr(self.config, attr):
+                    delattr(self.config, attr)
+            self.pbar.close()
+            self.pbar = None
 
 
 @dataclass