update csv export to child only, no parents

Author: cdoyle-temple

lib/tasks/detect_duplicates.rake

@@ -26,6 +26,8 @@ namespace :duplicates do
     log.call("Scanning assets with matching checksums...")
     log.call("Processing in batches of #{batch_size}...")
 
+    main_volume_names = %w[Deposit Media-Repository]
+
     duplicate_checksums = IsilonAsset.where("NULLIF(TRIM(file_checksum), '') IS NOT NULL")
                                      .group(:file_checksum)
                                      .having("COUNT(*) > 1")
@@ -44,7 +46,8 @@ namespace :duplicates do
     end
 
     written = 0
-    CSV.open(output_path, "w", write_headers: true, headers: [ "FullPath" ]) do |csv|
+    headers = [ "File", "Path", "Checksum", "File Size" ]
+    CSV.open(output_path, "w", write_headers: true, headers: headers) do |csv|
       duplicate_checksums.each_slice(batch_size) do |checksum_batch|
         batch_started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         checksum_batch.each_with_index do |checksum, index|
@@ -67,13 +70,20 @@ namespace :duplicates do
           DuplicateGroupMembership.insert_all(rows) if rows.any?
           IsilonAsset.where(id: asset_ids).update_all(has_duplicates: true)
 
-          IsilonAsset.where(id: asset_ids)
-                     .includes(parent_folder: :volume)
-                     .find_each do |asset|
+          main_scope = IsilonAsset.joins(parent_folder: :volume)
+                                  .where(file_checksum: checksum, volumes: { name: main_volume_names })
+          outside_scope = IsilonAsset.joins(parent_folder: :volume)
+                                     .where(file_checksum: checksum)
+                                     .where.not(volumes: { name: main_volume_names })
+
+          next unless main_scope.exists?
+          next unless outside_scope.exists?
+
+          outside_scope.includes(parent_folder: :volume).find_each do |asset|
             full_path = build_full_path.call(asset)
             next unless full_path
 
-            csv << [ full_path ]
+            csv << [ asset.isilon_name, full_path, checksum, asset.file_size ]
             written += 1
           end
 

spec/tasks/duplicates_rake_spec.rb

@@ -18,6 +18,13 @@
     let!(:media_volume) { create(:volume, name: "Media-Repository") }
     let!(:deposit_folder) { create(:isilon_folder, volume: deposit_volume, full_path: "/Deposit/project") }
     let!(:media_folder) { create(:isilon_folder, volume: media_volume, full_path: "/Media-Repository/project") }
+    let(:export_path) { Rails.root.join("log/isilon-duplicate-paths.csv") }
+    let(:detect_log_path) { Rails.root.join("log/isilon-duplicates-detect.log") }
+
+    after do
+      File.delete(export_path) if File.exist?(export_path)
+      File.delete(detect_log_path) if File.exist?(detect_log_path)
+    end
 
     it "groups assets with matching checksums" do
       original = create(:isilon_asset, parent_folder: deposit_folder, file_checksum: "abc", file_size: "100")
@@ -37,6 +44,30 @@
       expect(IsilonAsset.where(has_duplicates: true).count).to eq(4)
       expect(IsilonAsset.where(has_duplicates: false).count).to eq(2)
     end
+
+    it "exports child rows with checksum and file size for checksums shared across main and outside volumes" do
+      other_volume = create(:volume, name: "Other")
+      other_folder = create(:isilon_folder, volume: other_volume, full_path: "/Other/project")
+
+      create(:isilon_asset, parent_folder: deposit_folder, isilon_path: "/project/main.txt", isilon_name: "main.txt", file_checksum: "abc", file_size: "100")
+      create(:isilon_asset, parent_folder: media_folder, isilon_path: "/project/main2.txt", isilon_name: "main2.txt", file_checksum: "abc", file_size: "100")
+      outside_asset = create(:isilon_asset, parent_folder: other_folder, isilon_path: "/project/out.txt", isilon_name: "out.txt", file_checksum: "abc", file_size: "100")
+      create(:isilon_asset, parent_folder: other_folder, isilon_path: "/project/solo.txt", isilon_name: "solo.txt", file_checksum: "xyz", file_size: "100")
+
+      Rake::Task["duplicates:detect"].invoke
+
+      exported = CSV.read(export_path, headers: true)
+      child_row = exported.find { |row| row["File"] == "out.txt" }
+      solo_row = exported.find { |row| row["File"] == "solo.txt" }
+
+      expect(child_row).to be_present
+      expect(child_row["Path"]).to eq("/Other/project/out.txt")
+      expect(child_row["Checksum"]).to eq("abc")
+      expect(child_row["File Size"]).to eq("100")
+      expect(exported.find { |row| row["File"] == "main.txt" }).to be_nil
+      expect(exported.find { |row| row["File"] == "main2.txt" }).to be_nil
+      expect(solo_row).to be_nil
+    end
   end
 
   describe "duplicates:clear" do