Switch USE_SHIFTER env var name; remove scary log

JobRunner/DockerRunner.py

@@ -140,7 +140,7 @@ def _pull_and_run(self, image, env, labels, vols, cgroup_parent=None):
             image_id = self.docker.images.get(name=image).id
             self.pulled[image] = image_id
         except docker.errors.ImageNotFound as e:
-            self.logger.error(f"{e}")
+            self.logger.log(f"Image not found locally, will attempt to pull. Error was:\n{e}")
 
         try:
             # If no tag is specified, will return a list

JobRunner/JobRunner.py

@@ -368,22 +368,20 @@ def run(self):
             raise e
 
         try:
-            config = self.ee2.list_config()
+            ee2_config = self.ee2.list_config()
         except Exception as e:
-            self.logger.error("Failed to config . Exiting.")
+            self.logger.error("Failed to get config. Exiting.")
             raise e
 
-        if "USE_SHIFTER" in os.environ:
-            # Replace URLs for NERSC environment if set to "https://services.kbase.us"
-            old_url = "https://services.kbase.us"
-            new_url = "https://kbase.us"
-            for key, value in config.items():
-                if isinstance(value, str) and old_url in value:
-                    config[key] = value.replace(old_url, new_url)
+        if self.config.use_external_urls:
+            for key, value in ee2_config.items():
+                if isinstance(value, str) and Config.PROD_INTERNAL_URL_BASE in value:
+                    ee2_config[key] = value.replace(
+                        Config.PROD_INTERNAL_URL_BASE, Config.PROD_EXTERNAL_URL_BASE
+                    )
 
-        # config["job_id"] = self.job_id
         self.logger.log(
-            f"Server version of Execution Engine: {config.get('ee.server.version')}"
+            f"Server version of Execution Engine: {ee2_config.get('ee.server.version')}"
         )
 
         # Update job as started and log it
@@ -436,9 +434,9 @@ def run(self):
         # Note the self.config is not used, its the ee2 config we just grabbed and modified
         # TODO Try except for when submit or watch failure happens and correct finishjob call
         self._submit(
-            config=config, job_id=self.job_id, job_params=job_params, subjob=False
+            config=ee2_config, job_id=self.job_id, job_params=job_params, subjob=False
         )
-        output = self._watch(config)
+        output = self._watch(ee2_config)
         self.cbs.terminate()
         self.logger.log("Job is done")
 

JobRunner/callback_server.py

@@ -177,7 +177,7 @@ def _handle_submit(app, module, method, data, token):
             f"No more than {app.config['maxjobs']} concurrently running methods are allowed"
         )
     if module != "special":
-        # "special" denotes the method call does something unusual. The module is not registered 
+        # "special" denotes the method call does something unusual. The module is not registered
         # in the catalog. Not clear how to reasonably test this case.
         # Validate the module and version using the CatalogCache before submitting the job.
         # If there is an error with the module lookup, return the error response immediately.

JobRunner/config.py

@@ -34,6 +34,10 @@ def _get_admin_token():
 
 
 class Config:
+
+    PROD_INTERNAL_URL_BASE = "https://services.kbase.us"
+    PROD_EXTERNAL_URL_BASE = "https://kbase.us"
+
     def __init__(
             self,
             workdir=None,
@@ -43,7 +47,13 @@ def __init__(
             max_tasks: Union[int, None] = None,
     ):
         self.job_id = job_id
+        self.use_external_urls = os.environ.get("USE_EXTERNAL_URLS", "false").lower() == "true"
         self.base_url = os.environ.get(_KB_BASE_URL, "https://ci.kbase.us/services/")
+        if self.use_external_urls and self.base_url.startswith(self.PROD_INTERNAL_URL_BASE):
+            # internal urls are only accessible inside the KBase firewall
+            self.base_url = self.base_url.replace(
+                self.PROD_INTERNAL_URL_BASE, self.PROD_EXTERNAL_URL_BASE
+            )
         self.ee2_url = None
         self.debug = False
         self.cgroup = None

scripts/jobrunner.py

@@ -94,11 +94,6 @@ def main():
         sys.exit(1)
     ee2_suffix = ee2_url.split("/")[-1]
     base_url = ee2_url.rstrip(ee2_suffix)
-    # Replace URLs for NERSC environment if set to "https://services.kbase.us"
-    old_url = "https://services.kbase.us"
-    new_url = "https://kbase.us"
-    if "USE_SHIFTER" in os.environ and base_url.startswith(old_url):
-        base_url = base_url.replace(old_url, new_url)
 
     config = Config(workdir=os.getcwd(), job_id=job_id, base_url=base_url)
     if not os.path.exists(config.workdir):