From 419005da47a671029939562c6989a677d41297ea Mon Sep 17 00:00:00 2001 From: Laurence Cowton Date: Wed, 12 Feb 2020 17:54:38 +0000 Subject: [PATCH 1/5] raise custom exception to clarify FileNotFooundError caused by job failing --- faculty_distributed/manager.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/faculty_distributed/manager.py b/faculty_distributed/manager.py index 1ee3c1e..21195c5 100644 --- a/faculty_distributed/manager.py +++ b/faculty_distributed/manager.py @@ -8,6 +8,14 @@ from faculty import client +class JobOutputNotFoundError(Exception): + """Raised when the output of a job has not been saved. This error is likely + to have been raised due to a failed job. + """ + + pass + + class FacultyJobExecutor: """ Run generic python function in parallel on Faculty Jobs @@ -107,11 +115,18 @@ def _collect_output(self, args_sequence): """ out = [] for i in range(len(args_sequence)): - with open( - os.path.join(self.tmpdir, "output/out_{}.pkl".format(i)), "rb" - ) as f: - out.append(cloudpickle.load(f)) - + filepath = os.path.join(self.tmpdir, "output/out_{}.pkl".format(i)) + try: + with open(filepath, "rb",) as f: + out.append(cloudpickle.load(f)) + except FileNotFoundError: + raise JobOutputNotFoundError( + "No such file: '{}'. This error is likely to have been " + "raised due to a failed job. Check the Jobs history for " + "further details.".format( + filepath + ) + ) return out def _make_dirs(self): From 44c8fb9e2d59d4a8b5a9ae7bd8a84f7ecfc546f2 Mon Sep 17 00:00:00 2001 From: Laurence Cowton Date: Wed, 12 Feb 2020 17:59:11 +0000 Subject: [PATCH 2/5] run black --- faculty_distributed/manager.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/faculty_distributed/manager.py b/faculty_distributed/manager.py index 21195c5..98c321e 100644 --- a/faculty_distributed/manager.py +++ b/faculty_distributed/manager.py @@ -123,9 +123,7 @@ def _collect_output(self, args_sequence): raise JobOutputNotFoundError( "No such file: '{}'. This error is likely to have been " "raised due to a failed job. Check the Jobs history for " - "further details.".format( - filepath - ) + "further details.".format(filepath) ) return out From afb6a310b9a0384c4ed67c0dd48e6fe4b73c9d67 Mon Sep 17 00:00:00 2001 From: Laurence Cowton Date: Wed, 12 Feb 2020 18:02:14 +0000 Subject: [PATCH 3/5] update black version in tox --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 563b64d..e9accf5 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,6 @@ commands = [testenv:black] skip_install = True deps = - black==19.3b0 + black==19.10b0 commands = black {posargs:--check .} From bee46303c7c7b0b9226c8b1b6971576573ecf5cc Mon Sep 17 00:00:00 2001 From: Laurence Cowton Date: Wed, 12 Feb 2020 18:09:30 +0000 Subject: [PATCH 4/5] fix python 2 issue due to FileNotFoundError not exisiting in python 2.7 --- faculty_distributed/manager.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/faculty_distributed/manager.py b/faculty_distributed/manager.py index 98c321e..d98b271 100644 --- a/faculty_distributed/manager.py +++ b/faculty_distributed/manager.py @@ -119,12 +119,15 @@ def _collect_output(self, args_sequence): try: with open(filepath, "rb",) as f: out.append(cloudpickle.load(f)) - except FileNotFoundError: - raise JobOutputNotFoundError( - "No such file: '{}'. This error is likely to have been " - "raised due to a failed job. Check the Jobs history for " - "further details.".format(filepath) - ) + except OSError as e: + if isinstance(e, FileNotFoundError): + raise JobOutputNotFoundError( + "No such file: '{}'. This error is likely to have " + "been raised due to a failed job. Check the Jobs " + "history for further details.".format(filepath) + ) + else: + raise e return out def _make_dirs(self): From 619f5b4fc57f9dbb2cad5e8da0c0feda691fd8ba Mon Sep 17 00:00:00 2001 From: Laurence Cowton Date: Wed, 12 Feb 2020 18:26:45 +0000 Subject: [PATCH 5/5] drop python 2 support --- .travis.yml | 3 --- faculty_distributed/manager.py | 15 ++++++--------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index fd26fed..d5b914b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,9 +7,6 @@ matrix: - name: Check black formatting python: 3.7 env: TOXENV=black - - name: Check flake8 on Python 2 - python: 2.7 - env: TOXENV=flake8 - name: Check flake8 on Python 3 python: 3.7 env: TOXENV=flake8 diff --git a/faculty_distributed/manager.py b/faculty_distributed/manager.py index d98b271..98c321e 100644 --- a/faculty_distributed/manager.py +++ b/faculty_distributed/manager.py @@ -119,15 +119,12 @@ def _collect_output(self, args_sequence): try: with open(filepath, "rb",) as f: out.append(cloudpickle.load(f)) - except OSError as e: - if isinstance(e, FileNotFoundError): - raise JobOutputNotFoundError( - "No such file: '{}'. This error is likely to have " - "been raised due to a failed job. Check the Jobs " - "history for further details.".format(filepath) - ) - else: - raise e + except FileNotFoundError: + raise JobOutputNotFoundError( + "No such file: '{}'. This error is likely to have been " + "raised due to a failed job. Check the Jobs history for " + "further details.".format(filepath) + ) return out def _make_dirs(self):