Fix fallback function value in case of simulator failure

src/ensemble/ensemble.py

@@ -277,7 +277,7 @@ def calc_prediction(self, enX=None, save_prediction=None):
 
             # Parallelization on HPC using SLURM
             elif self.sim.input_dict.get('hpc', False): # Run prediction in parallel on hpc
-                en_pred = self.run_on_HPC(enX, batch_size=nparallel)
+                en_pred = self.run_on_HPC(enX, batch_size=nparallel, save_prediction=save_prediction)
 
             # Parallelization on local machine using p_map      
             else:
@@ -412,6 +412,11 @@ def run_on_HPC(self, enX, batch_size=None, **kwargs):
                 else:
                     en_pred.append(False)
                 self.sim.remove_folder(member_i)
+
+            # Store partial batch results if needed
+            part_pred_data = dtools.en_pred_to_pred_data(en_pred)
+            if 'save_prediction' in kwargs and kwargs['save_prediction']:
+                np.savez(kwargs['save_prediction'] +'.npz', **{'pred_data': part_pred_data})
 
         return en_pred
 

src/popt/loop/ensemble_base.py

@@ -139,7 +139,7 @@ def function(self, x, *args, **kwargs):
         else:
             func_values = np.inf  # the simulations have crashed
 
-        if len(x.shape) == 1:
+        if len(x.shape) == 1 and func_values != np.inf:
             self.stateF = func_values
         else:
             self.enF = func_values 

src/popt/loop/optimize.py

@@ -191,7 +191,9 @@ def run_loop(self):
                     self.logger(f'─────> EPF-EnOpt: maximum epf iterations reached')  # print epf info
                     break
                 #p = np.abs(previous_state-self.xk) / (np.abs(previous_state) + 1.0e-9)
-                p = np.mean(self.epf['penalty'] / self.epf['r'])  # the penalty term (without r)
+                if self.epf['penalty'].size == 0:
+                    raise ValueError('EPF penalty is empty; cannot compute convergence criterion.')
+                p = np.mean(self.epf['penalty']) / self.epf['r']  # the penalty term (without r)
                 conv_crit = self.epf['conv_crit']
                 if p > conv_crit:
                     epf_not_converged = True
@@ -208,7 +210,7 @@ def run_loop(self):
                     r = self.epf['r']
                     self.logger(f'─────> EPF-EnOpt: {self.epf_iteration}, {r} (outer iteration, penalty factor)')  # print epf info
                 else:
-                    self.logger(f'─────> EPF-EnOpt: converged, no variables changed more than {conv_crit*100} %')  # print epf info
+                    self.logger(f'─────> EPF-EnOpt: converged, penalty term larger than {conv_crit}')  # print epf info
                     final_obj_no_penalty = str( round( float( np.mean(self.fun(self.xk)) ),4) )
                     self.logger(f'─────> EPF-EnOpt: objective value without penalty = {final_obj_no_penalty}') # print epf info
     def save(self):