Merge pull request #31 from CU-CloudCollab/spot-reservation

added spot market functions to the library

Author: sbower

cucloud.gemspec

@@ -26,6 +26,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'aws-sdk', '~> 2'
   spec.add_dependency 'uuid', '~> 2.3'
+  spec.add_dependency 'descriptive_statistics', '~> 2.5'
 
   spec.add_development_dependency 'bundler', '~> 1.11'
   spec.add_development_dependency 'coveralls'

lib/cucloud.rb

@@ -1,4 +1,5 @@
 require 'aws-sdk'
+require 'descriptive_statistics'
 
 # Main Cucloud Module namespace and defaults
 module Cucloud
@@ -15,6 +16,7 @@ module Cucloud
   require 'cucloud/rds_utils'
   require 'cucloud/lambda_utils'
   require 'cucloud/cfn_utils'
+  require 'cucloud/utilities'
 
   # This is the default region API calls are made against
   DEFAULT_REGION = 'us-east-1'.freeze

lib/cucloud/ec2_utils.rb

@@ -11,6 +11,10 @@ class Ec2Utils
     WAITER_MAX_ATTEMPS = 240
     # Delay between calls used by waiter to check status
     WAITER_DELAY = 15
+    # Two weeks in hours
+    TWO_WEEKS = 336
+    # Default OS to use
+    DEFAULT_OS = 'Linux/UNIX'.freeze
 
     def initialize(ec2_client = Aws::EC2::Client.new, ssm_utils = Cucloud::SSMUtils.new)
       @ec2 = ec2_client
@@ -253,5 +257,64 @@ def find_ebs_snapshots(options = {})
       end
       found_snapshots
     end
+
+    # Get a recommendation for a spot bid request.  Given an instance type and
+    # OS we will grab data from a period specified, default is from two weeks to now,
+    # and calculate recommendations for the AZs in the current region
+    # @param instance_type [String] Insrance type to get bid for
+    # @param os [String] OS you whish to run, default linux
+    # @param num_hours [Integer] How many hours to look back, default two weeks
+    # @return [Hash] Reccomendations by region, empty if no viable recommendations
+    def best_spot_bid_price(instance_type, os = DEFAULT_OS, num_hours = TWO_WEEKS)
+      price_history_by_az = {}
+      recommendations = {}
+
+      options = {
+        end_time: Time.now.utc,
+        instance_types: [
+          instance_type
+        ],
+        product_descriptions: [
+          os
+        ],
+        start_time: (Time.now - num_hours * 60).utc
+      }
+
+      loop do
+        price_history = @ec2.describe_spot_price_history(options)
+        price_history.spot_price_history.each do |price|
+          price_history_by_az[price.availability_zone] = [] unless price_history_by_az[price.availability_zone]
+          price_history_by_az[price.availability_zone].push(price.spot_price.to_f)
+        end
+
+        break if price_history.next_token.nil? || price_history.next_token.empty?
+        options[:next_token] = price_history.next_token
+      end
+
+      price_history_by_az.each do |key, data|
+        stats = data.descriptive_statistics
+        next unless stats[:number] > 30
+        confidence_interval = Cucloud::Utilities.confidence_interval_99(
+          stats[:mean],
+          stats[:standard_deviation],
+          stats[:number]
+        )
+        recommendations[key] = confidence_interval[1]
+      end
+
+      recommendations
+    end
+
+    # Make spot instance request
+    # @param options [Hash] Options to provide to the API
+    # see http://docs.aws.amazon.com/sdkforruby/api/Aws/EC2/Client.html#request_spot_instances-instance_method
+    # @return [Hash] Description of the spot request
+    def make_spot_instance_request(options)
+      spot_requests = @ec2.request_spot_instances(options)
+      request_ids = [spot_requests.spot_instance_requests[0].spot_instance_request_id]
+
+      @ec2.wait_until(:spot_instance_request_fulfilled, spot_instance_request_ids: request_ids)
+      @ec2.describe_spot_instance_requests(spot_instance_request_ids: request_ids)
+    end
   end
 end

lib/cucloud/utilities.rb

@@ -0,0 +1,39 @@
+module Cucloud
+  # Utilities class - for basice shared utilities
+  class Utilities
+    # Z Score to calculate 99% confidence interval
+    Z_SCORE_99 = 2.576
+    # Z Score to calculate 99% confidence interval
+    Z_SCORE_96 = 1.96
+
+    # Calculate 99% confidence interval
+    # @param mean [Float] sample mean
+    # @param stdev [Float] sample standard deviation
+    # @param sample_size [Integer] sample size
+    # @return [Array] Two element array representing the computed confidence interval
+    def self.confidence_interval_99(mean, stdev, sample_size)
+      confidence_interval(mean, stdev, sample_size, Z_SCORE_99)
+    end
+
+    # Calculate 95% confidence interval
+    # @param mean [Float] sample mean
+    # @param stdev [Float] sample standard deviation
+    # @param sample_size [Integer] sample size
+    # @return [Array] Two element array representing the computed confidence interval
+    def self.confidence_interval_95(mean, stdev, sample_size)
+      confidence_interval(mean, stdev, sample_size, Z_SCORE_96)
+    end
+
+    private_class_method
+
+    # Calculate confidence interval for given zscore
+    # @param mean [Float] sample mean
+    # @param stdev [Float] sample standard deviation
+    # @param sample_size [Integer] sample size
+    # @return [Array] Two element array representing the computed confidence interval
+    def self.confidence_interval(mean, stdev, sample_size, zscore)
+      delta = zscore * stdev / Math.sqrt(sample_size - 1)
+      [mean - delta, mean + delta]
+    end
+  end
+end

lib/cucloud/version.rb

@@ -1,5 +1,5 @@
 module Cucloud
   # Disable mutable constant warning - freezing this oddly breaks bundler
   # rubocop:disable Style/MutableConstant
-  VERSION = '0.7.5'
+  VERSION = '0.7.6'
 end

spec/ec2_utils_spec.rb

@@ -412,6 +412,88 @@
       expect(ec_util.get_instance_name('i-2')).to be_nil
     end
 
+    describe 'make_spot_instance_request' do
+      it 'should make a spot instnace request' do
+        ec2_client.stub_responses(
+          :describe_spot_instance_requests,
+          spot_instance_requests: [{
+            status: {
+              code: 'fulfilled'
+            }
+          }]
+        )
+
+        ec2_client.stub_responses(
+          :request_spot_instances,
+          spot_instance_requests: [{
+            spot_instance_request_id: 'sir-1212121'
+          }]
+        )
+
+        options = {
+          instance_count: 1,
+          launch_specification: {
+
+            image_id: 'ami-275ffe31',
+            instance_type: 'm3.medium'
+          },
+          spot_price: '0.016',
+          type: 'one-time'
+        }
+
+        expect { ec_util.make_spot_instance_request(options) }.not_to raise_error
+      end
+    end
+
+    describe 'best_spot_bid_price' do
+      it 'should return a list of bid recommendations' do
+        ec2_client.stub_responses(
+          :describe_spot_price_history,
+          JSON.parse(File.read(File.join(File.dirname(__FILE__), '/fixtures/bid_history.json')), symbolize_names: true)
+        )
+
+        bid_prices = ec_util.best_spot_bid_price('m3.medium')
+        expect(bid_prices).to match_array([
+                                            ['us-west-1a', 0.08244842404174188],
+                                            ['us-west-1c', 0.07917263606261282]
+                                          ])
+      end
+
+      it 'should return a list of bid recommendations (paginate)' do
+        ec2_client.stub_responses(
+          :describe_spot_price_history,
+          [
+            {
+              next_token: '1123123',
+              spot_price_history: [
+                {
+                  availability_zone: 'us-west-1a',
+                  instance_type: 'm3.medium',
+                  product_description: 'Linux/UNIX (Amazon VPC)',
+                  spot_price: '0.080000',
+                  timestamp: Time.parse('2014-01-06T04:32:53.000Z')
+                }
+              ]
+            },
+            {
+              next_token: '',
+              spot_price_history: [
+                {
+                  availability_zone: 'us-west-1a',
+                  instance_type: 'm3.medium',
+                  product_description: 'Linux/UNIX (Amazon VPC)',
+                  spot_price: '0.080000',
+                  timestamp: Time.parse('2014-01-06T04:32:53.000Z')
+                }
+              ]
+            }
+          ]
+        )
+
+        expect { ec_util.best_spot_bid_price('m3.medium') }.not_to raise_error
+      end
+    end
+
     describe '#instances_to_patch_by_tag' do
       it 'should run without an error with no valid targets' do
         expect { ec_util.instances_to_patch_by_tag }.not_to raise_error