Report generation performance improvements (#310)

* Clean up controller logic by extracting to `data()` function.

* Add basic test coverage.

* Check all files instead of directories, because of how they're generated into a random structure.

* Actual implementation has to be using carbon too.

* Windows, yuck.

* This stuff is required on Page.

* Chunking overhaul, almost there.

* Defer validating of pages until after all yaml pages are generated.

* Test chunking.

* Suppress pending page results errors when running queue workers.

* Ensure we only trigger generation once, even with async queue workers.

* These aren’t being used anywhere. They are defined on report.

* Massively improve performance around validating unique titles and descriptions.

* Cleanup and more performance improvements.

* Cache key cleanup.

* Only cache `toArray()` when generated and WITH pages.

* Make queue chunk size configurable.

* Fix resource deleter refs issue.

* Ensure caches are cleared when creating and deleting, in case IDs get reused.

* Update test.

Author: jesseleite

config/seo-pro.php

@@ -43,4 +43,8 @@
         'excluded_sites' => [],
     ],
 
+    'reports' => [
+        'queue_chunk_size' => 1000,
+    ],
+
 ];

resources/views/reports/index.blade.php

@@ -28,9 +28,9 @@
                         @can('delete seo reports')
                             <td class="float-right">
                                 <dropdown-list>
-                                    <dropdown-item :text="__('seo-pro::messages.delete_report')" class="warning" @click="$refs.deleter.confirm()">
+                                    <dropdown-item :text="__('seo-pro::messages.delete_report')" class="warning" @click="$refs.deleter_{{ $report->id() }}.confirm()">
                                         <resource-deleter
-                                            ref="deleter"
+                                            ref="deleter_{{ $report->id() }}"
                                             resource-title="Report"
                                             route="{{ cp_route('seo-pro.reports.destroy', $report->id()) }}"
                                             redirect="{{ cp_route('seo-pro.reports.index') }}"

src/Http/Controllers/ReportController.php

@@ -36,7 +36,7 @@ public function show(Request $request, $id)
         $report = Report::find($id);
 
         if ($request->ajax()) {
-            return $this->showOrGenerateReport($report);
+            return $report->data();
         }
 
         return view('seo-pro::reports.show', ['report' => $report]);
@@ -48,15 +48,4 @@ public function destroy($id)
 
         return Report::find($id)->delete();
     }
-
-    private function showOrGenerateReport($report)
-    {
-        if ($report->status() === 'pending') {
-            $report = $report->queueGenerate();
-        } elseif ($report->isLegacyReport()) {
-            $report = $report->updateLegacyReport();
-        }
-
-        return $report->withPages();
-    }
 }

src/Reporting/Chunk.php

@@ -0,0 +1,91 @@
+<?php
+
+namespace Statamic\SeoPro\Reporting;
+
+use Illuminate\Support\Facades\Cache;
+use Statamic\Facades\Data;
+use Statamic\Facades\File;
+use Statamic\Facades\YAML;
+use Statamic\SeoPro\Cascade;
+use Statamic\SeoPro\GetsSectionDefaults;
+use Statamic\SeoPro\SiteDefaults;
+
+class Chunk
+{
+    use GetsSectionDefaults;
+
+    public $id;
+    public $contentIds;
+    public $report;
+
+    public function __construct($id, $contentIds, Report $report)
+    {
+        $this->id = $id;
+        $this->contentIds = $contentIds;
+        $this->report = $report;
+    }
+
+    protected function folderPath()
+    {
+        return $this->report->chunksFolder()."/{$this->id}";
+    }
+
+    protected function yamlPath()
+    {
+        return $this->folderPath().'/chunk.yaml';
+    }
+
+    public function save()
+    {
+        File::put($this->yamlPath(), YAML::dump([
+            'ids' => $this->contentIds,
+        ]));
+
+        return $this;
+    }
+
+    public function queueGenerate()
+    {
+        $ids = $this->contentIds;
+
+        dispatch(function () use ($ids) {
+            $this->generate($ids);
+        });
+    }
+
+    protected function generate($ids)
+    {
+        $content = Cache::get($this->report->cacheKey(Report::CONTENT_CACHE_KEY_SUFFIX));
+
+        foreach ($ids as $id) {
+            $this->createPage($content[$id] ?? Data::find($id))->save();
+        }
+
+        File::delete($this->folderPath());
+
+        if ($this->wasLastChunk()) {
+            $this->report->finalize();
+        }
+    }
+
+    protected function wasLastChunk()
+    {
+        return File::getFolders($this->report->chunksFolder())->isEmpty();
+    }
+
+    protected function createPage($content)
+    {
+        if ($content->value('seo') === false || is_null($content->uri())) {
+            return;
+        }
+
+        $data = (new Cascade)
+            ->with(SiteDefaults::load()->augmented())
+            ->with($this->getAugmentedSectionDefaults($content))
+            ->with($content->augmentedValue('seo')->value())
+            ->withCurrent($content)
+            ->get();
+
+        return new Page($content->id(), $data, $this->report);
+    }
+}

src/Reporting/Page.php

@@ -5,26 +5,20 @@
 use Statamic\Facades\Data;
 use Statamic\Facades\File;
 use Statamic\Facades\YAML;
+use Statamic\Support\Arr;
 
 class Page
 {
-    protected $report;
+    protected $id;
     protected $data;
+    protected $report;
     protected $results;
-    protected $id;
 
-    protected $rules = [
-        Rules\UniqueTitleTag::class,
-        Rules\UniqueMetaDescription::class,
-        Rules\NoUnderscoresInUrl::class,
-        Rules\ThreeSegmentUrls::class,
-    ];
-
-    public function setData($data)
+    public function __construct($id, $data, Report $report)
     {
-        $this->data = collect($data);
-
-        return $this;
+        $this->id = $id;
+        $this->data = $data;
+        $this->report = $report;
     }
 
     public function setResults($results)
@@ -34,13 +28,6 @@ public function setResults($results)
         return $this;
     }
 
-    public function setReport(Report $report)
-    {
-        $this->report = $report;
-
-        return $this;
-    }
-
     public function report()
     {
         return $this->report;
@@ -75,11 +62,15 @@ public function validate()
 
     public function get($key)
     {
-        return $this->data->get($key);
+        return Arr::get($this->data, $key);
     }
 
     public function status()
     {
+        if (! $this->results) {
+            return 'pending';
+        }
+
         $status = 'pass';
 
         foreach ($this->getRuleResults() as $result) {
@@ -99,6 +90,10 @@ public function getRuleResults()
     {
         $results = collect();
 
+        if (! $this->results) {
+            return $results;
+        }
+
         foreach ($this->results as $class => $array) {
             $class = "Statamic\\SeoPro\\Reporting\\Rules\\$class";
             $rule = new $class;
@@ -129,18 +124,11 @@ public function id()
         return $this->id;
     }
 
-    public function setId($id)
-    {
-        $this->id = $id;
-
-        return $this;
-    }
-
     public function save()
     {
         $data = [
             'id' => $this->id,
-            'data' => $this->data->all(),
+            'data' => $this->data,
             'results' => $this->results,
         ];