Merge pull request #64 from brettshollenberger/aws-specs

rc99

Author: brettshollenberger

.ruby-gemset

@@ -0,0 +1 @@
+easy_ml

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    easy_ml (0.2.0.pre.rc98)
+    easy_ml (0.2.0.pre.rc99)
       activerecord
       activerecord-import (~> 1.8.1)
       activesupport

app/models/easy_ml/cleaner.rb

@@ -29,8 +29,6 @@ def clean
       end
     end
 
-    private
-
     def files_to_keep_for_dir(dir)
       files_to_keep.map(&:to_s).select { |f| f.start_with?(dir.to_s) }
     end
@@ -55,24 +53,28 @@ def active_models
         end
     end
 
+    def test_mode?
+      Rails.env.test?
+    end
+
     def model_files_to_keep
-      if Rails.env.test?
+      if test_mode?
         []
       else
         active_models.map(&:model_file).compact.map(&:full_path).uniq
       end
     end
 
     def dataset_files_to_keep
-      if Rails.env.test?
+      if test_mode?
         []
       else
-        EasyML::Dataset.all.flat_map(&:files).uniq
+        EasyML::Dataset.all.flat_map(&:files).uniq + active_models.map(&:dataset).flat_map(&:files).uniq
       end
     end
 
     def datasource_files_to_keep
-      if Rails.env.test?
+      if test_mode?
         Dir.glob(EasyML::Engine.root_dir.glob("datasources/**/*.{csv,parquet}")).uniq
       else
         EasyML::Datasource.all.flat_map(&:files).uniq

app/models/easy_ml/model.rb

@@ -559,7 +559,6 @@ def actually_deploy
       dataset.upload_remote_files
       model_snapshot = snapshot
 
-      Thread.current[:deploying] = true
       # Prepare the model to be retrained (reset values so they don't conflict with our snapshotted version)
       bump_version(force: true)
       dataset.bump_versions(version)

easy_ml-0.2.0.pre.rc99.gem

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module EasyML
-  VERSION = "0.2.0-rc98"
+  VERSION = "0.2.0-rc99"
 
   module Version
   end

lib/easy_ml/version.rb

@@ -236,6 +236,55 @@ def relative_dir(dir)
       Timecop.return
     end
 
+    it "does not clean up active datasets/models after deploy" do
+      @t1 = EasyML::Support::UTC.parse("2025-01-01").beginning_of_day
+      Timecop.freeze(@t1)
+
+      mock_s3_upload
+      model.save
+      model.unlock!
+
+      @t2 = EasyML::Support::UTC.parse("2025-01-02").beginning_of_day
+      Timecop.freeze(@t2)
+
+      model.train(async: false)
+      model.deploy(async: false)
+      model_v1 = model.current_version
+      dataset_v1 = model_v1.dataset
+
+      model.dataset.columns.where(name: "Age").update_all(hidden: true)
+      model.dataset.refresh
+
+      @t3 = EasyML::Support::UTC.parse("2025-01-03").beginning_of_day
+      Timecop.freeze(@t3)
+
+      model.train(async: false)
+      model.deploy(async: false)
+      model_v2 = model.current_version
+      dataset_v2 = model_v2.dataset
+
+      cleaner = EasyML::Cleaner.new
+      allow(cleaner).to receive(:test_mode?).and_return(false)
+
+      expect(cleaner.active_models).to include(model_v2)
+      expect(cleaner.active_models).to include(model)
+      expect(cleaner.active_models).to_not include(model_v1)
+
+      expect(cleaner.model_files_to_keep).to include(model_v2.model_file.full_path)
+      expect(cleaner.model_files_to_keep).to include(model.model_file.full_path)
+      expect(cleaner.model_files_to_keep).to_not include(model_v1.model_file.full_path)
+
+      dataset_v2.files.each do |file|
+        expect(cleaner.dataset_files_to_keep).to include(file)
+      end
+      dataset.files.each do |file|
+        expect(cleaner.dataset_files_to_keep).to include(file)
+      end
+      dataset_v1.files.each do |file|
+        expect(cleaner.dataset_files_to_keep).to_not include(file)
+      end
+    end
+
     it "uses deployed version for prediction" do
       mock_s3_upload
       @time = EasyML::Support::UTC.now