Merge pull request #91 from govigilant/develop

2025.12.2

Author: VincentBean

packages/crawler/src/Actions/CrawlUrl.php

@@ -4,12 +4,14 @@
 
 use Illuminate\Http\Client\ConnectionException;
 use Illuminate\Http\Client\Response;
+use Illuminate\Support\Arr;
 use Illuminate\Support\Facades\Gate;
 use Illuminate\Support\Facades\Http;
 use Illuminate\Support\Str;
 use Vigilant\Core\Services\TeamService;
 use Vigilant\Crawler\Enums\State;
 use Vigilant\Crawler\Models\CrawledUrl;
+use Vigilant\Crawler\Models\IgnoredUrl;
 use Vigilant\Crawler\Notifications\RatelimitedNotification;
 
 class CrawlUrl
@@ -105,9 +107,28 @@ public function crawl(CrawledUrl $url, int $try = 0): void
             ];
         }
 
+        $existingLinks = CrawledUrl::query()
+            ->where('crawler_id', '=', $url->crawler_id)
+            ->whereIn('url_hash', Arr::pluck($queuedLinks, 'url_hash'))
+            ->pluck('url_hash')
+            ->all();
+
+        $queuedLinks = array_filter($queuedLinks, function (array $record) use ($existingLinks): bool {
+            return ! in_array($record['url_hash'], $existingLinks, true);
+        });
+
         if ($queuedLinks !== []) {
             $timestamp = now();
             $records = [];
+            $ignoredHashes = [];
+
+            if ($url->crawler_id !== null) {
+                $ignoredHashes = IgnoredUrl::query()
+                    ->where('crawler_id', '=', $url->crawler_id)
+                    ->whereIn('url_hash', array_keys($queuedLinks))
+                    ->pluck('url_hash')
+                    ->all();
+            }
 
             foreach ($queuedLinks as $record) {
                 $records[] = [
@@ -117,6 +138,7 @@ public function crawl(CrawledUrl $url, int $try = 0): void
                     'url_hash' => $record['url_hash'],
                     'url' => $record['url'],
                     'found_on_id' => $record['found_on_id'],
+                    'ignored' => in_array($record['url_hash'], $ignoredHashes, true),
                     'crawled' => false,
                     'created_at' => $timestamp,
                     'updated_at' => $timestamp,

packages/crawler/src/Actions/ImportSitemaps.php

@@ -7,6 +7,7 @@
 use Mtownsend\XmlToArray\XmlToArray;
 use Vigilant\Crawler\Models\CrawledUrl;
 use Vigilant\Crawler\Models\Crawler;
+use Vigilant\Crawler\Models\IgnoredUrl;
 
 class ImportSitemaps
 {
@@ -73,16 +74,31 @@ protected function storeUrls(Crawler $crawler, Collection $urls): void
                 continue;
             }
 
+            $timestamp = now();
+            $urlHashes = $newUrls->map(fn ($url): string => md5($url));
+            $ignoredHashes = $urlHashes->isEmpty()
+                ? []
+                : IgnoredUrl::query()
+                    ->where('crawler_id', '=', $crawler->id)
+                    ->whereIn('url_hash', $urlHashes->all())
+                    ->pluck('url_hash')
+                    ->all();
+
             $crawler->urls()->insert(
-                $newUrls->map(fn ($url): array => [
-                    'uuid' => (new CrawledUrl)->newUniqueId(),
-                    'crawler_id' => $crawler->id,
-                    'team_id' => $crawler->team_id,
-                    'url' => $url,
-                    'url_hash' => md5($url),
-                    'created_at' => now(),
-                    'updated_at' => now(),
-                ])->toArray()
+                $newUrls->map(function ($url) use ($crawler, $timestamp, $ignoredHashes): array {
+                    $hash = md5($url);
+
+                    return [
+                        'uuid' => (new CrawledUrl)->newUniqueId(),
+                        'crawler_id' => $crawler->id,
+                        'team_id' => $crawler->team_id,
+                        'url' => $url,
+                        'url_hash' => $hash,
+                        'ignored' => in_array($hash, $ignoredHashes, true),
+                        'created_at' => $timestamp,
+                        'updated_at' => $timestamp,
+                    ];
+                })->toArray()
             );
         }
     }

packages/crawler/src/Jobs/StartCrawlerJob.php

@@ -0,0 +1,37 @@
+<?php
+
+namespace Vigilant\Crawler\Jobs;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Vigilant\Core\Services\TeamService;
+use Vigilant\Crawler\Actions\StartCrawler;
+use Vigilant\Crawler\Models\Crawler;
+
+class StartCrawlerJob implements ShouldBeUnique, ShouldQueue
+{
+    use Dispatchable;
+    use InteractsWithQueue;
+    use Queueable;
+    use SerializesModels;
+
+    public function __construct(public Crawler $crawler)
+    {
+        $this->onQueue(config('crawler.queue'));
+    }
+
+    public function handle(StartCrawler $starter, TeamService $teamService): void
+    {
+        $teamService->setTeamById($this->crawler->team_id);
+        $starter->start($this->crawler);
+    }
+
+    public function uniqueId(): int
+    {
+        return $this->crawler->id;
+    }
+}

packages/crawler/src/Livewire/Crawler/Dashboard.php

@@ -30,7 +30,7 @@ public function render(): mixed
 
         return view($view, [
             'total_url_count' => $crawler->totalUrlCount(),
-            'issue_count' => $crawler->issueCount(),
+            'issue_count' => $crawler->issueCount() ?? 0,
             'ignored_count' => $crawler->urls()->where('ignored', '=', true)->count(),
             'nextRun' => $crawler->enabled ? $nextRun->diffForHumans() : __('Crawler disabled'),
         ]);

packages/crawler/src/Observers/CrawlerObserver.php

@@ -2,18 +2,13 @@
 
 namespace Vigilant\Crawler\Observers;
 
-use Vigilant\Crawler\Actions\StartCrawler;
+use Vigilant\Crawler\Jobs\StartCrawlerJob;
 use Vigilant\Crawler\Models\Crawler;
 
 class CrawlerObserver
 {
-    public function __construct(
-        protected StartCrawler $startCrawler,
-    ) {
-    }
-
     public function created(Crawler $crawler): void
     {
-        $this->startCrawler->start($crawler);
+        StartCrawlerJob::dispatch($crawler);
     }
 }

packages/crawler/tests/Actions/ProcessCrawlerStateTest.php

@@ -3,39 +3,15 @@
 namespace Vigilant\Crawler\Tests\Actions;
 
 use Illuminate\Support\Facades\Event;
-use Mockery\MockInterface;
 use PHPUnit\Framework\Attributes\Test;
 use Vigilant\Crawler\Actions\ProcessCrawlerState;
-use Vigilant\Crawler\Actions\StartCrawler;
 use Vigilant\Crawler\Enums\State;
 use Vigilant\Crawler\Events\CrawlerFinishedEvent;
 use Vigilant\Crawler\Models\Crawler;
 use Vigilant\Crawler\Tests\TestCase;
 
 class ProcessCrawlerStateTest extends TestCase
 {
-    #[Test]
-    public function it_processes_pending_state(): void
-    {
-        /** @var Crawler $crawler */
-        $crawler = Crawler::query()->create([
-            'start_url' => 'vigilant',
-            'state' => State::Pending,
-            'schedule' => '0 0 * * *',
-        ]);
-
-        $this->mock(StartCrawler::class, function (MockInterface $mock): void {
-            $mock->shouldReceive('start')->andReturn()->once();
-        });
-
-        $crawler->forceFill(['state' => State::Pending])->save();
-        $crawler->refresh();
-
-        /** @var ProcessCrawlerState $action */
-        $action = app(ProcessCrawlerState::class);
-        $action->process($crawler);
-    }
-
     #[Test]
     public function it_processes_crawling_finished_state(): void
     {

packages/healthchecks/src/Observers/HealthcheckObserver.php

@@ -3,6 +3,7 @@
 namespace Vigilant\Healthchecks\Observers;
 
 use Illuminate\Support\Str;
+use Vigilant\Healthchecks\Jobs\CheckHealthcheckJob;
 use Vigilant\Healthchecks\Models\Healthcheck;
 
 class HealthcheckObserver
@@ -13,4 +14,9 @@ public function creating(Healthcheck $healthcheck): void
             $healthcheck->token = Str::random(32);
         }
     }
+
+    public function created(Healthcheck $healthcheck): void
+    {
+        CheckHealthcheckJob::dispatch($healthcheck);
+    }
 }

packages/healthchecks/tests/Actions/CheckMetricTest.php

@@ -3,6 +3,7 @@
 namespace Vigilant\Healthchecks\Tests\Actions;
 
 use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Bus;
 use PHPUnit\Framework\Attributes\Test;
 use Vigilant\Healthchecks\Actions\CheckMetric;
 use Vigilant\Healthchecks\Enums\Type;
@@ -22,6 +23,8 @@ public function it_does_nothing_when_no_metrics_exist(): void
         MetricIncreasingNotification::fake();
         MetricSpikeNotification::fake();
 
+        Bus::fake();
+
         $healthcheck = Healthcheck::query()->create([
             'domain' => 'example.com',
             'type' => Type::Laravel,

packages/lighthouse/src/Models/LighthouseMonitor.php

@@ -12,6 +12,7 @@
 use Vigilant\Core\Scopes\TeamScope;
 use Vigilant\Sites\Models\Site;
 use Vigilant\Users\Observers\TeamObserver;
+use Vigilant\Lighthouse\Observers\LighthouseMonitorObserver;
 
 /**
  * @property int $id
@@ -29,7 +30,7 @@
  * @property ?Site $site
  * @property Collection<int, LighthouseResult> $lighthouseResults
  */
-#[ObservedBy([TeamObserver::class])]
+#[ObservedBy([TeamObserver::class, LighthouseMonitorObserver::class])]
 #[ScopedBy([TeamScope::class])]
 class LighthouseMonitor extends Model
 {

packages/lighthouse/src/Observers/LighthouseMonitorObserver.php

@@ -0,0 +1,14 @@
+<?php
+
+namespace Vigilant\Lighthouse\Observers;
+
+use Vigilant\Lighthouse\Jobs\RunLighthouseJob;
+use Vigilant\Lighthouse\Models\LighthouseMonitor;
+
+class LighthouseMonitorObserver
+{
+    public function created(LighthouseMonitor $monitor): void
+    {
+        RunLighthouseJob::dispatch($monitor);
+    }
+}