Adjust backfill requests to latest spec

Author: oneiros

debug_fasp/db/migrate/20250103144206_replace_cursor_with_fulfilled_flag.fasp_data_sharing.rb

@@ -0,0 +1,7 @@
+# This migration comes from fasp_data_sharing (originally 20250103141700)
+class ReplaceCursorWithFulfilledFlag < ActiveRecord::Migration[8.0]
+  def change
+    remove_column :fasp_data_sharing_backfill_requests, :cursor, :string
+    add_column :fasp_data_sharing_backfill_requests, :fulfilled, :boolean, null: false, default: false
+  end
+end

debug_fasp/db/schema.rb

@@ -20,7 +20,7 @@ def response_authentication_headers(server, status, body)
     headers.merge(signature.to_h)
   end
 
-  def stub_request_to(server, method, path, response_status, response_body)
+  def stub_request_to(server, method, path, response_status, response_body = "")
     response_body = encode_body(response_body)
     response_headers = {
       "content-type" => "application/json"

fasp_base/lib/fasp_base/integration_test_helper.rb

@@ -9,21 +9,29 @@ class BackfillRequest < ApplicationRecord
     validates :max_count, numericality: { only_integer: true }
 
     class << self
-      def make(server, category:, max_count: 100, cursor: nil)
+      def make(server, category:, max_count: 100)
         response = FaspBase::Request.new(server)
           .post("/data_sharing/v0/backfill_requests", body: {
             category:,
-            maxCount: max_count,
-            cursor:
+            maxCount: max_count
           }.compact
         )
 
         remote_id = response["backfillRequest"]["id"]
         create!(
           fasp_base_server: server, remote_id:, category:,
-          max_count:, cursor:
+          max_count:
         )
       end
     end
+
+    def continue!
+      response = FaspBase::Request.new(fasp_base_server)
+        .post("/data_sharing/v0/backfill_requests/#{remote_id}/continuation")
+    end
+
+    def mark_as_fulfilled!
+      update!(fulfilled: true)
+    end
   end
 end

fasp_data_sharing/app/models/fasp_data_sharing/backfill_request.rb

@@ -30,7 +30,6 @@ class FaspDataSharing::Event
   attribute :source
   attribute :category, :string
   attribute :event_type, :string
-  attribute :cursor, :string
   attribute :more_objects_available, :boolean
   attribute :object_uris
 
@@ -56,8 +55,7 @@ def process
   def update_backfill_request
     return unless backfill_event?
 
-    @source_record.update(cursor:) if cursor
-    @source_record.destroy if more_objects_available == false
+    @source_record.mark_as_fulfilled! unless more_objects_available
   end
 
   def source_is_known

fasp_data_sharing/app/models/fasp_data_sharing/event.rb

@@ -0,0 +1,6 @@
+class ReplaceCursorWithFulfilledFlag < ActiveRecord::Migration[8.0]
+  def change
+    remove_column :fasp_data_sharing_backfill_requests, :cursor, :string
+    add_column :fasp_data_sharing_backfill_requests, :fulfilled, :boolean, null: false, default: false
+  end
+end

fasp_data_sharing/db/migrate/20250103141700_replace_cursor_with_fulfilled_flag.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.0].define(version: 2024_12_11_140048) do
+ActiveRecord::Schema[8.0].define(version: 2025_01_03_141700) do
   create_table "fasp_base_servers", force: :cascade do |t|
     t.string "base_url", null: false
     t.integer "user_id", null: false
@@ -20,16 +20,16 @@
     t.string "fasp_remote_id"
     t.string "public_key_pem"
     t.string "registration_completion_uri"
-    t.index [ "base_url" ], name: "index_fasp_base_servers_on_base_url", unique: true
-    t.index [ "user_id" ], name: "index_fasp_base_servers_on_user_id"
+    t.index ["base_url"], name: "index_fasp_base_servers_on_base_url", unique: true
+    t.index ["user_id"], name: "index_fasp_base_servers_on_user_id"
   end
 
   create_table "fasp_base_users", force: :cascade do |t|
     t.string "email", null: false
     t.string "password_digest"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.index [ "email" ], name: "index_fasp_base_users_on_email", unique: true
+    t.index ["email"], name: "index_fasp_base_users_on_email", unique: true
   end
 
   create_table "fasp_data_sharing_actors", force: :cascade do |t|
@@ -43,10 +43,10 @@
     t.string "remote_id", null: false
     t.string "category", null: false
     t.integer "max_count"
-    t.string "cursor"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.index [ "fasp_base_server_id" ], name: "idx_on_fasp_base_server_id_0f3fe7f51e"
+    t.boolean "fulfilled", default: false, null: false
+    t.index ["fasp_base_server_id"], name: "idx_on_fasp_base_server_id_0f3fe7f51e"
   end
 
   create_table "fasp_data_sharing_subscriptions", force: :cascade do |t|
@@ -61,7 +61,7 @@
     t.integer "threshold_replies"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.index [ "fasp_base_server_id" ], name: "index_fasp_data_sharing_subscriptions_on_fasp_base_server_id"
+    t.index ["fasp_base_server_id"], name: "index_fasp_data_sharing_subscriptions_on_fasp_base_server_id"
   end
 
   add_foreign_key "fasp_base_servers", "fasp_base_users", column: "user_id"

fasp_data_sharing/test/dummy/db/schema.rb

@@ -1,6 +1,6 @@
 require "test_helper"
 
-class ActorTest < ActionDispatch::IntegrationTest
+class AnnouncementsTest < ActionDispatch::IntegrationTest
   include FaspBase::IntegrationTestHelper
 
   setup do
@@ -81,7 +81,7 @@ class ActorTest < ActionDispatch::IntegrationTest
     assert_response :success
   end
 
-  test "with a matching backfill request it enqueues jobs and updates the cursor" do
+  test "with a matching backfill request it enqueues jobs" do
     backfill_request = fasp_data_sharing_backfill_requests(:accounts)
     payload = {
       source: {
@@ -90,7 +90,7 @@ class ActorTest < ActionDispatch::IntegrationTest
         }
       },
       category: "account",
-      cursor: "efgh",
+      moreObjectsAvailable: true,
       objectUris: [
         "https://fediverse-server.example.com/@example/2342",
         "https://fediverse-server.example.com/@other/8726"
@@ -104,11 +104,10 @@ class ActorTest < ActionDispatch::IntegrationTest
     end
 
     assert_response :success
-    assert_equal "efgh", backfill_request.reload.cursor
   end
 
 
-  test "with a matching backfill request and no more objects to get it deletes the backfill request" do
+  test "with a matching backfill request and no more objects to get it marks the backfill request as fulfilled" do
     backfill_request = fasp_data_sharing_backfill_requests(:accounts)
     payload = {
       source: {
@@ -127,9 +126,11 @@ class ActorTest < ActionDispatch::IntegrationTest
     uri = fasp_data_sharing.fasp_data_sharing_v0_announcements_url
     headers = request_authentication_headers(@server, :post, uri, payload)
 
-    assert_difference -> { FaspDataSharing::BackfillRequest.count }, -1 do
+    assert_changes -> { backfill_request.fulfilled? } do
       assert_enqueued_jobs 2, only: FaspDataSharing::ProcessAccountBackfillJob do
         post fasp_data_sharing.fasp_data_sharing_v0_announcements_path, as: :json, params: payload, headers: headers
+
+        backfill_request.reload
       end
     end
 

fasp_data_sharing/test/integration/announcements_test.rb

@@ -32,5 +32,31 @@ class BackfillRequestTest < ActiveSupport::TestCase
       assert_equal "account", backfill_request.category
       assert_equal 90, backfill_request.max_count
     end
+
+    test "continuing an existing backfill request" do
+      stubbed_request = stub_request_to(@server, :post, "/data_sharing/v0/backfill_requests/672/continuation", 204)
+      backfill_request = BackfillRequest.create(
+        fasp_base_server: @server,
+        remote_id: "672",
+        category: "content",
+        max_count: 100
+      )
+      backfill_request.continue!
+
+      assert_requested(stubbed_request)
+    end
+
+    test "marking as fulfilled" do
+      backfill_request = BackfillRequest.create(
+        fasp_base_server: @server,
+        remote_id: "672",
+        category: "content",
+        max_count: 100
+      )
+
+      assert_changes -> { backfill_request.fulfilled? } do
+        backfill_request.mark_as_fulfilled!
+      end
+    end
   end
 end