Merge pull request #63 from brettshollenberger/aws-specs

Aws specs

Author: brettshollenberger

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    easy_ml (0.2.0.pre.rc97)
+    easy_ml (0.2.0.pre.rc98)
       activerecord
       activerecord-import (~> 1.8.1)
       activesupport

app/models/easy_ml/dataset.rb

@@ -667,8 +667,6 @@ def load_dataset
     end
 
     def upload_remote_files
-      return if !needs_refresh?
-
       processed.upload.tap do
         features.each(&:upload_remote_files)
         features.each(&:save)

app/models/easy_ml/deploy.rb

@@ -65,8 +65,14 @@ def actually_deploy
 
           EasyML::Deploy.transaction do
             update(model_history_id: self.model_version.id, snapshot_id: self.model_version.snapshot_id, status: :success)
-            model.retraining_runs.where(status: :deployed).update_all(status: :success)
-            retraining_run.update(model_history_id: self.model_version.id, snapshot_id: self.model_version.snapshot_id, deploy_id: id, status: :deployed,)
+            model.retraining_runs.where(status: :deployed).update_all(status: :success, is_deploying: false)
+            retraining_run.update(
+              model_history_id: self.model_version.id,
+              snapshot_id: self.model_version.snapshot_id,
+              deploy_id: id,
+              status: :deployed,
+              is_deploying: false
+            )
           end
 
           model_version.tap do

app/models/easy_ml/model_file.rb

@@ -36,6 +36,10 @@ def synced_file
       )
     end
 
+    def s3_prefix
+      EasyML::Configuration.model_s3_path(model.name)
+    end
+
     def root_dir
       Pathname.new(model.root_dir)
     end

app/models/easy_ml/settings.rb

@@ -21,6 +21,16 @@ class Settings < ActiveRecord::Base
 
     validates :storage, inclusion: { in: %w[file s3] }, if: -> { storage.present? }
 
+    after_initialize :set_defaults, if: -> { new_record? }
+
+    def set_defaults
+      self.s3_prefix ||= "easy_ml"
+    end
+
+    def s3_prefix
+      read_attribute(:s3_prefix) || "easy_ml"
+    end
+
     TIMEZONES = [
       { value: "America/New_York", label: "Eastern Time" },
       { value: "America/Chicago", label: "Central Time" },

easy_ml-0.2.0.pre.rc98.gem

@@ -60,5 +60,17 @@ def db_settings
     def db_settings
       @db_settings ||= EasyML::Settings.first_or_create
     end
+
+    def self.s3_path_for(type, name)
+      File.join(instance.s3_prefix, type.to_s, name.to_s)
+    end
+
+    def self.model_s3_path(model_name)
+      s3_path_for("models", model_name.parameterize.gsub("-", "_"))
+    end
+
+    def self.dataset_s3_path(relative_path)
+      s3_path_for("datasets", relative_path)
+    end
   end
 end

lib/easy_ml/configuration.rb

@@ -125,6 +125,10 @@ def download
         synced_directory.download
       end
 
+      def remote_files
+        synced_directory.remote_files
+      end
+
       private
 
       def root

lib/easy_ml/data/dataset_manager.rb

@@ -46,7 +46,9 @@ def attributes
         end
 
         def s3_prefix
-          File.join("datasets", dir.split("datasets").last)
+          EasyML::Configuration.dataset_s3_path(
+            dir.split("datasets").last
+          )
         end
 
         def synced_directory

lib/easy_ml/data/splits/file_split.rb

@@ -36,6 +36,19 @@ def bump_version(original_version, version)
       )
     end
 
+    # Expose S3 information for testing and verification
+    def s3_prefix
+      EasyML::Configuration.dataset_s3_path(
+        File.join(feature_dir.split("datasets").last, "compacted")
+      )
+    end
+
+    def s3_key(filename = nil)
+      filename ||= File.basename(files.first) if files.any?
+      return nil unless filename
+      File.join(s3_prefix, filename)
+    end
+
     private
 
     def batch_size
@@ -49,9 +62,5 @@ def feature_dir
         feature&.name&.parameterize&.gsub("-", "_")
       )
     end
-
-    def s3_prefix
-      File.join("datasets", feature_dir.split("datasets").last)
-    end
   end
 end