added the freeze automation to the client object (#120)

* added the freeze automation to the client object, pulled from CLI

* added test cases for freeze

* fixed imports

* fixed directory collision

Author: zeryx

Algorithmia/CLI.py

@@ -2,7 +2,6 @@
 import os
 from Algorithmia.errors import DataApiError
 from Algorithmia.algo_response import AlgoResponse
-from Algorithmia.util import md5_for_file, md5_for_str
 import json, re, requests, six
 import toml
 import shutil
@@ -244,28 +243,7 @@ def cat(self, path, client):
 
     # algo freeze
     def freezeAlgo(self, client, manifest_path="model_manifest.json"):
-        if os.path.exists(manifest_path):
-            with open(manifest_path, 'r') as f:
-                manifest_file = json.load(f)
-            manifest_file['timestamp'] = str(time())
-            required_files = manifest_file['required_files']
-            optional_files = manifest_file['optional_files']
-            for i in range(len(required_files)):
-                uri = required_files[i]['source_uri']
-                local_file = client.file(uri).getFile(as_path=True)
-                md5_checksum = md5_for_file(local_file)
-                required_files[i]['md5_checksum'] = md5_checksum
-            for i in range(len(optional_files)):
-                uri = required_files[i]['source_uri']
-                local_file = client.file(uri).getFile(as_path=True)
-                md5_checksum = md5_for_file(local_file)
-                required_files[i]['md5_checksum'] = md5_checksum
-            lock_md5_checksum = md5_for_str(str(manifest_file))
-            manifest_file['lock_checksum'] = lock_md5_checksum
-            with open('model_manifest.json.freeze', 'w') as f:
-                json.dump(manifest_file, f)
-        else:
-            print("Expected to find a model_manifest.json file, none was discovered in working directory")
+        client.freeze(manifest_path)
 
     # algo cp <src> <dest>
     def cp(self, src, dest, client):

Algorithmia/algorithm.py

@@ -8,10 +8,12 @@
 from Algorithmia.errors import ApiError, ApiInternalError, raiseAlgoApiError
 from enum import Enum
 from algorithmia_api_client.rest import ApiException
-from algorithmia_api_client import CreateRequest, UpdateRequest, VersionRequest, Details, Settings, SettingsMandatory, SettingsPublish, \
+from algorithmia_api_client import CreateRequest, UpdateRequest, VersionRequest, Details, Settings, SettingsMandatory, \
+    SettingsPublish, \
     CreateRequestVersionInfo, VersionInfo, VersionInfoPublish
 
-OutputType = Enum('OutputType','default raw void')
+OutputType = Enum('OutputType', 'default raw void')
+
 
 class Algorithm(object):
     def __init__(self, client, algoRef):
@@ -32,7 +34,7 @@ def __init__(self, client, algoRef):
             raise ValueError('Invalid algorithm URI: ' + algoRef)
 
     def set_options(self, timeout=300, stdout=False, output=OutputType.default, **query_parameters):
-        self.query_parameters = {'timeout':timeout, 'stdout':stdout}
+        self.query_parameters = {'timeout': timeout, 'stdout': stdout}
         self.output_type = output
         self.query_parameters.update(query_parameters)
         return self
@@ -42,7 +44,8 @@ def create(self, details={}, settings={}, version_info={}):
         detailsObj = Details(**details)
         settingsObj = SettingsMandatory(**settings)
         createRequestVersionInfoObj = CreateRequestVersionInfo(**version_info)
-        create_parameters = {"name": self.algoname, "details": detailsObj, "settings": settingsObj, "version_info": createRequestVersionInfoObj}
+        create_parameters = {"name": self.algoname, "details": detailsObj, "settings": settingsObj,
+                             "version_info": createRequestVersionInfoObj}
         create_request = CreateRequest(**create_parameters)
         try:
             # Create Algorithm
@@ -57,7 +60,8 @@ def update(self, details={}, settings={}, version_info={}):
         detailsObj = Details(**details)
         settingsObj = Settings(**settings)
         createRequestVersionInfoObj = CreateRequestVersionInfo(**version_info)
-        update_parameters = {"details": detailsObj, "settings": settingsObj, "version_info": createRequestVersionInfoObj}
+        update_parameters = {"details": detailsObj, "settings": settingsObj,
+                             "version_info": createRequestVersionInfoObj}
         update_request = UpdateRequest(**update_parameters)
         try:
             # Update Algorithm
@@ -70,9 +74,10 @@ def update(self, details={}, settings={}, version_info={}):
     # Publish an algorithm
     def publish(self, details={}, settings={}, version_info={}):
         publish_parameters = {"details": details, "settings": settings, "version_info": version_info}
-        url = "/v1/algorithms/"+self.username+"/"+self.algoname + "/versions"
+        url = "/v1/algorithms/" + self.username + "/" + self.algoname + "/versions"
         print(publish_parameters)
-        api_response = self.client.postJsonHelper(url, publish_parameters, parse_response_as_json=True, **self.query_parameters)
+        api_response = self.client.postJsonHelper(url, publish_parameters, parse_response_as_json=True,
+                                                  **self.query_parameters)
         return api_response
         # except ApiException as e:
         #     error_message = json.loads(e.body)
@@ -81,7 +86,8 @@ def publish(self, details={}, settings={}, version_info={}):
     def builds(self, limit=56, marker=None):
         try:
             if marker is not None:
-                api_response = self.client.manageApi.get_algorithm_builds(self.username, self.algoname, limit=limit, marker=marker)
+                api_response = self.client.manageApi.get_algorithm_builds(self.username, self.algoname, limit=limit,
+                                                                          marker=marker)
             else:
                 api_response = self.client.manageApi.get_algorithm_builds(self.username, self.algoname, limit=limit)
             return api_response
@@ -109,11 +115,10 @@ def get_build_logs(self, build_id):
             raise raiseAlgoApiError(error_message)
 
     def build_logs(self):
-        url = '/v1/algorithms/'+self.username+'/'+self.algoname+'/builds'
+        url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/builds'
         response = json.loads(self.client.getHelper(url).content.decode('utf-8'))
         return response
 
-
     def get_scm_status(self):
         try:
             api_response = self.client.manageApi.get_algorithm_scm_connection_status(self.username, self.algoname)
@@ -157,7 +162,6 @@ def versions(self, limit=None, marker=None, published=None, callable=None):
             error_message = json.loads(e.body)
             raise raiseAlgoApiError(error_message)
 
-
     # Compile an algorithm
     def compile(self):
         try:
@@ -176,25 +180,26 @@ def pipe(self, input1):
         elif self.output_type == OutputType.void:
             return self._postVoidOutput(input1)
         else:
-            return AlgoResponse.create_algo_response(self.client.postJsonHelper(self.url, input1, **self.query_parameters))
+            return AlgoResponse.create_algo_response(
+                self.client.postJsonHelper(self.url, input1, **self.query_parameters))
 
     def _postRawOutput(self, input1):
-            # Don't parse response as json
-            self.query_parameters['output'] = 'raw'
-            response = self.client.postJsonHelper(self.url, input1, parse_response_as_json=False, **self.query_parameters)
-            # Check HTTP code and throw error as needed
-            if response.status_code == 400:
-                # Bad request
-                raise ApiError(response.text)
-            elif response.status_code == 500:
-                raise ApiInternalError(response.text)
-            else:
-                return response.text
+        # Don't parse response as json
+        self.query_parameters['output'] = 'raw'
+        response = self.client.postJsonHelper(self.url, input1, parse_response_as_json=False, **self.query_parameters)
+        # Check HTTP code and throw error as needed
+        if response.status_code == 400:
+            # Bad request
+            raise ApiError(response.text)
+        elif response.status_code == 500:
+            raise ApiInternalError(response.text)
+        else:
+            return response.text
 
     def _postVoidOutput(self, input1):
-            self.query_parameters['output'] = 'void'
-            responseJson = self.client.postJsonHelper(self.url, input1, **self.query_parameters)
-            if 'error' in responseJson:
-                raise ApiError(responseJson['error']['message'])
-            else:
-                return AsyncResponse(responseJson)
+        self.query_parameters['output'] = 'void'
+        responseJson = self.client.postJsonHelper(self.url, input1, **self.query_parameters)
+        if 'error' in responseJson:
+            raise ApiError(responseJson['error']['message'])
+        else:
+            return AsyncResponse(responseJson)

Algorithmia/client.py

@@ -6,12 +6,13 @@
 from Algorithmia.datafile import DataFile, LocalDataFile, AdvancedDataFile
 from Algorithmia.datadirectory import DataDirectory, LocalDataDirectory, AdvancedDataDirectory
 from algorithmia_api_client import Configuration, DefaultApi, ApiClient
-
+from Algorithmia.util import md5_for_file, md5_for_str
 from tempfile import mkstemp
 import atexit
 import json, re, requests, six, certifi
 import tarfile
 import os
+from time import time
 
 
 class Client(object):
@@ -343,6 +344,30 @@ def exit_handler(self):
         except OSError as e:
             print(e)
 
+    # Used by CI/CD automation for freezing model manifest files, and by the CLI for manual freezing
+    def freeze(self, manifest_path, manifest_output_dir="."):
+        if os.path.exists(manifest_path):
+            with open(manifest_path, 'r') as f:
+                manifest_file = json.load(f)
+            manifest_file['timestamp'] = str(time())
+            required_files = manifest_file['required_files']
+            optional_files = manifest_file['optional_files']
+            for i in range(len(required_files)):
+                uri = required_files[i]['source_uri']
+                local_file = self.file(uri).getFile(as_path=True)
+                md5_checksum = md5_for_file(local_file)
+                required_files[i]['md5_checksum'] = md5_checksum
+            for i in range(len(optional_files)):
+                uri = required_files[i]['source_uri']
+                local_file = self.file(uri).getFile(as_path=True)
+                md5_checksum = md5_for_file(local_file)
+                required_files[i]['md5_checksum'] = md5_checksum
+            lock_md5_checksum = md5_for_str(str(manifest_file))
+            manifest_file['lock_checksum'] = lock_md5_checksum
+            with open(manifest_output_dir+'/'+'model_manifest.json.freeze', 'w') as f:
+                json.dump(manifest_file, f)
+        else:
+            print("Expected to find a model_manifest.json file, none was discovered in working directory")
 
 def isJson(myjson):
     try:

Test/client_test.py

@@ -397,6 +397,10 @@ def test_algorithm_programmatic_create_process(self):
 
             self.assertEqual(response.version_info.semantic_version, "0.1.0", "information is incorrect")
 
+        def test_algo_freeze(self):
+            self.regular_client.freeze("Test/resources/manifests/example_manifest.json", "Test/resources/manifests")
+
+
 
 if __name__ == '__main__':
     unittest.main()

Test/resources/manifests/example_manifest.json

@@ -0,0 +1,29 @@
+{
+  "required_files" : [
+      { "name": "squeezenet",
+      "source_uri": "data://AlgorithmiaSE/image_cassification_demo/squeezenet1_1-f364aa15.pth",
+      "fail_on_tamper": true,
+      "metadata": {
+        "dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
+      }
+    },
+    {
+      "name": "labels",
+      "source_uri": "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json",
+      "fail_on_tamper": true,
+      "metadata": {
+        "dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
+      }
+    }
+  ],
+  "optional_files": [
+      {
+        "name": "mobilenet",
+        "source_uri": "data://AlgorithmiaSE/image_cassification_demo/mobilenet_v2-b0353104.pth",
+        "fail_on_tamper": false,
+        "metadata": {
+            "dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
+          }
+      }
+  ]
+}