Bugfix: Reindex job fix (CMS 6)

src/DataObject/DataObjectFetcher.php

@@ -23,6 +23,10 @@ class DataObjectFetcher implements DocumentFetcherInterface
 
     private int $offset = 0;
 
+    private static string $fetch_sort = 'ID';
+
+    private static string $fetch_sort_direction = 'ASC';
+
     public function __construct(string $class)
     {
         if (!is_subclass_of($class, DataObject::class)) {
@@ -74,7 +78,14 @@ public function incrementOffsetDown(): void
      */
     public function fetch(): array
     {
-        $list = $this->createDataList($this->getBatchSize(), $this->getOffset());
+        // get configurable sort options
+        $sortBy = static::config()->get('fetch_sort') ?? 'ID';
+        $sortDirection = static::config()->get('fetch_sort_direction') ?? 'ASC';
+
+        // sort (default by ID) to ensure consistent ordering across batches
+        $list = $this->createDataList($this->getBatchSize(), $this->getOffset())
+            ->sort($sortBy, $sortDirection);
+
         $docs = [];
 
         foreach ($list as $record) {

tests/DataObject/DataObjectFetcherTest.php

@@ -8,6 +8,7 @@
 use SilverStripe\Forager\Tests\Fake\DataObjectFake;
 use SilverStripe\Forager\Tests\Fake\DataObjectSubclassFake;
 use SilverStripe\Forager\Tests\Fake\DataObjectSubclassFakeShouldNotIndex;
+use SilverStripe\Forager\Tests\Fake\PageFakeVersioned;
 use SilverStripe\Forager\Tests\SearchServiceTestTrait;
 
 class DataObjectFetcherTest extends SapphireTest
@@ -23,6 +24,7 @@ class DataObjectFetcherTest extends SapphireTest
      */
     protected static $extra_dataobjects = [
         DataObjectFake::class,
+        PageFakeVersioned::class,
     ];
 
     public function testFetch(): void
@@ -63,6 +65,52 @@ public function testFetch(): void
         $this->assertCount(2, $documents);
     }
 
+    /**
+     * This tests that we fetch all documents when processed in batches.
+     */
+    public function testFetchBatch(): void
+    {
+        // create pages
+        $createPageCount = 100;
+
+        for ($i = 0; $i < $createPageCount; $i++) {
+            $dataobject = PageFakeVersioned::create();
+            $dataobject->Title = sprintf('FetchTestPage');
+            // added to verify that all pages are set regardless of the sort order
+            $dataobject->Sort = 1;
+            $dataobject->write();
+            $dataobject->publishSingle();
+        }
+
+        $batchSize = 10;
+        $fetcher = DataObjectFetcher::create(PageFakeVersioned::class);
+        $totalDocuments = $fetcher->getTotalDocuments();
+
+        $fetchedDocumentCount = 0;
+        $fetchedDocumentIDs = [];
+
+        // keep fetching until we've fetched all documents, using the batch size and offset to get the next batch of
+        // documents each time
+        while ($fetchedDocumentCount < $totalDocuments) {
+            $documents = $fetcher->fetch($batchSize, $fetchedDocumentCount);
+
+            $fetchedDocumentCount += count($documents);
+
+            // collect all ids so that we can check everything has been fetched at the end of the test
+            $batchIDs = array_map(function (DataObjectDocument $document) {
+                return $document->getDataObject()->ID;
+            }, $documents);
+
+            $fetchedDocumentIDs = array_merge($fetchedDocumentIDs, array_values($batchIDs));
+        }
+
+        // only get unique ids so that we can check that all expected documents have been fetched
+        $fetchedDocumentIDs = array_unique($fetchedDocumentIDs);
+
+        // make sure we fetched all the documents
+        $this->assertCount($totalDocuments, $fetchedDocumentIDs);
+    }
+
     public function testTotalDocuments(): void
     {
         $fetcher = DataObjectFetcher::create(DataObjectFake::class);

tests/Fake/PageFakeVersioned.php

@@ -0,0 +1,20 @@
+<?php
+
+namespace SilverStripe\Forager\Tests\Fake;
+
+use Page;
+use SilverStripe\Dev\TestOnly;
+use SilverStripe\Forager\Extensions\SearchServiceExtension;
+use SilverStripe\Versioned\Versioned;
+
+class PageFakeVersioned extends Page implements TestOnly
+{
+
+    private static string $table_name = 'PageFakeVersioned';
+
+    private static array $extensions = [
+        SearchServiceExtension::class,
+        Versioned::class,
+    ];
+
+}