From 16b088b1696f2b906d4096c69239a71c07ce832a Mon Sep 17 00:00:00 2001 From: Adrien Letournel Date: Wed, 11 Feb 2026 17:34:37 +0100 Subject: [PATCH 1/2] chore(SYNTH-24202): improve verbosity for synthetics ressources --- .../model/synthetics_global_variables.py | 7 +++- ...synthetics_mobile_applications_versions.py | 33 +++++++++++++++++-- datadog_sync/model/synthetics_tests.py | 6 +++- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/datadog_sync/model/synthetics_global_variables.py b/datadog_sync/model/synthetics_global_variables.py index ed2cd5f8..5658b4e5 100644 --- a/datadog_sync/model/synthetics_global_variables.py +++ b/datadog_sync/model/synthetics_global_variables.py @@ -37,8 +37,13 @@ class SyntheticsGlobalVariables(BaseResource): destination_global_variables: Dict[str, Dict] = dict() async def get_resources(self, client: CustomClient) -> List[Dict]: + self.config.logger.debug( + "synthetics_global_variables: fetching from %s", self.resource_config.base_path + ) resp = await client.get(self.resource_config.base_path) - return resp["variables"] + variables = resp.get("variables", []) + self.config.logger.debug("synthetics_global_variables: got %d variables", len(variables)) + return variables async def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = None) -> Tuple[str, Dict]: if _id: diff --git a/datadog_sync/model/synthetics_mobile_applications_versions.py b/datadog_sync/model/synthetics_mobile_applications_versions.py index e05ecadd..10062b84 100644 --- a/datadog_sync/model/synthetics_mobile_applications_versions.py +++ b/datadog_sync/model/synthetics_mobile_applications_versions.py @@ -39,14 +39,43 @@ async def get_resources(self, client: CustomClient) -> List[Dict]: """ Mobile Application Versions don't have a list endpoint of their own """ + self.config.logger.debug( + "synthetics_mobile_applications_versions: fetching applications list from %s", + self.applications_path, + ) resp = await client.get(self.applications_path) + applications = resp.get("applications", []) + total_versions = sum(len(app.get("versions", [])) for app in applications) + self.config.logger.info( + "synthetics_mobile_applications_versions: %d applications, %d versions to fetch", + len(applications), + total_versions, + ) resources = [] - for application in resp["applications"]: - for version in application["versions"]: + progress_interval = max(1, total_versions // 20) # log at INFO roughly every 5% + for app_idx, application in enumerate(applications): + versions = application.get("versions", []) + for ver_idx, version in enumerate(versions): _id = version["id"] + current = len(resources) + 1 + self.config.logger.debug( + "synthetics_mobile_applications_versions: fetching version %s (%d/%d)", + _id, + current, + total_versions, + ) resource = await client.get(self.resource_config.base_path + f"/{_id}") resources.append(resource) + if current % progress_interval == 0 or current == total_versions: + self.config.logger.info( + "synthetics_mobile_applications_versions: progress %d/%d", + current, + total_versions, + ) + self.config.logger.info( + "synthetics_mobile_applications_versions: finished fetching %d versions", len(resources) + ) return resources async def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = None) -> Tuple[str, Dict]: diff --git a/datadog_sync/model/synthetics_tests.py b/datadog_sync/model/synthetics_tests.py index d16b0c94..b0651a1e 100644 --- a/datadog_sync/model/synthetics_tests.py +++ b/datadog_sync/model/synthetics_tests.py @@ -74,13 +74,17 @@ class SyntheticsTests(BaseResource): versions: List = [] async def get_resources(self, client: CustomClient) -> List[Dict]: + self.config.logger.debug("synthetics_tests: fetching list from %s", self.resource_config.base_path) resp = await client.get( self.resource_config.base_path, params=self.get_params, ) + tests = resp.get("tests", []) + self.config.logger.debug("synthetics_tests: got %d tests, fetching mobile application versions...", len(tests)) versions = SyntheticsMobileApplicationsVersions(self.config) self.versions = await versions.get_resources(client) - return resp["tests"] + self.config.logger.debug("synthetics_tests: got %d mobile app versions", len(self.versions)) + return tests async def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = None) -> Tuple[str, Dict]: source_client = self.config.source_client From 29a821dc8b9587ff319caff01d7df18d09df94aa Mon Sep 17 00:00:00 2001 From: Adrien Letournel Date: Wed, 11 Feb 2026 21:36:35 +0100 Subject: [PATCH 2/2] improve progress update when many dependent ressources --- ...synthetics_mobile_applications_versions.py | 8 +++++ datadog_sync/utils/workers.py | 29 +++++++++++++++---- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/datadog_sync/model/synthetics_mobile_applications_versions.py b/datadog_sync/model/synthetics_mobile_applications_versions.py index 10062b84..7f3c2129 100644 --- a/datadog_sync/model/synthetics_mobile_applications_versions.py +++ b/datadog_sync/model/synthetics_mobile_applications_versions.py @@ -52,8 +52,14 @@ async def get_resources(self, client: CustomClient) -> List[Dict]: total_versions, ) + # If import progress bar is active, add sub-steps so the bar advances during fetch + add_total = getattr(self.config, "_import_progress_add_total", None) + if callable(add_total): + add_total(total_versions) + resources = [] progress_interval = max(1, total_versions // 20) # log at INFO roughly every 5% + progress_update = getattr(self.config, "_import_progress_update", None) for app_idx, application in enumerate(applications): versions = application.get("versions", []) for ver_idx, version in enumerate(versions): @@ -67,6 +73,8 @@ async def get_resources(self, client: CustomClient) -> List[Dict]: ) resource = await client.get(self.resource_config.base_path + f"/{_id}") resources.append(resource) + if callable(progress_update): + progress_update(1) if current % progress_interval == 0 or current == total_versions: self.config.logger.info( "synthetics_mobile_applications_versions: progress %d/%d", diff --git a/datadog_sync/utils/workers.py b/datadog_sync/utils/workers.py index 50e0e224..e36e3ea2 100644 --- a/datadog_sync/utils/workers.py +++ b/datadog_sync/utils/workers.py @@ -95,13 +95,30 @@ async def schedule_workers(self, additional_coros: List = []) -> Future: async def schedule_workers_with_pbar(self, total, additional_coros: List = []) -> Future: self.pbar = tqdm(total=total) - self._shutdown_workers = False - with logging_redirect_tqdm(): - additional_coros.append(self._refresh_pbar()) - await self.schedule_workers(additional_coros) + # Allow long-running get_resources (e.g. mobile app versions) to update the bar + def _update(n: int = 1) -> None: + if self.pbar: + self.pbar.update(n) - self.pbar.close() - self.pbar = None + def _add_total(n: int) -> None: + if self.pbar: + self.pbar.total += n + self.pbar.refresh() + + setattr(self.config, "_import_progress_update", _update) + setattr(self.config, "_import_progress_add_total", _add_total) + + self._shutdown_workers = False + try: + with logging_redirect_tqdm(): + additional_coros.append(self._refresh_pbar()) + await self.schedule_workers(additional_coros) + finally: + for attr in ("_import_progress_update", "_import_progress_add_total"): + if hasattr(self.config, attr): + delattr(self.config, attr) + self.pbar.close() + self.pbar = None @dataclass