diff --git a/dimcli/core/api.py b/dimcli/core/api.py index aac0ad0..3863b50 100644 --- a/dimcli/core/api.py +++ b/dimcli/core/api.py @@ -14,6 +14,7 @@ import IPython.display from itertools import islice import urllib.parse +from requests.exceptions import HTTPError import pandas as pd @@ -193,11 +194,18 @@ def query(self, q, show_results=None, retry=0, verbose=None): if verbose: printDebug("ERROR LOG\n---\nQuery\n---\n" + str(q), "red") if verbose: printDebug("Response.header\n---\n" + str(response.headers), "red") if verbose: printDebug("Response.content\n---\n" +str(response.content), "red") + response.raise_for_status() + # raise_for_status() doesn't treat 2XX messages as an error but empty HTTP 202 + # messages have been observed. + # + # Any HTTP response that has made it here needs to trigger an exception or else we + # will return None, which can cause problem for calling code. + raise HTTPError(f"Unexpected response HTTP {response.status_code}", response=response) - def query_iterative(self, q, show_results=None, limit=1000, skip=0, pause=1.5, force=False, maxlimit=0, verbose=None, _tot_count_prev_query=0, _warnings_tot=None): + def query_iterative(self, q, show_results=None, limit=1000, skip=0, pause=1.5, force=False, maxlimit=0, verbose=None, retry=0, _tot_count_prev_query=0, _warnings_tot=None): """Runs a DSL query and then keep querying until all matching records have been extracted. The API returns a maximum of 1000 records per call. If a DSL query results in more than 1000 matches, it is possible to use pagination to get more results, up to 50k. @@ -224,7 +232,7 @@ def query_iterative(self, q, show_results=None, limit=1000, skip=0, pause=1.5, f The maximum number of records to extract in total. If 0, all available records are extracted, up to the API upper limit of 50k records per query. verbose : bool, default=False Verbose mode. - + retry: number of retries per individual request, when an error is encountered Returns ------- @@ -291,7 +299,7 @@ def query_iterative(self, q, show_results=None, limit=1000, skip=0, pause=1.5, f q2 = q + " limit %d skip %d" % (limit, skip) start = time.time() - res = self.query(q2, show_results=False, retry=0, verbose=False) + res = self.query(q2, show_results=False, retry=retry, verbose=False) end = time.time() elapsed = end - start @@ -314,7 +322,7 @@ def query_iterative(self, q, show_results=None, limit=1000, skip=0, pause=1.5, f tot = _tot_count_prev_query # when force=True, we have no current query stats new_skip = skip+limit - if tot > 0 and new_skip > tot: + if tot is not None and tot > 0 and new_skip > tot: new_skip = tot if verbose and tot: # if not first iteration t = "%.2f" % elapsed @@ -1011,4 +1019,4 @@ def __repr__(self): # 2019-12-17: for backward compatibility # remove once all notebooks code has been updated Result = DslDataset -Dataset = DslDataset \ No newline at end of file +Dataset = DslDataset