Skip to content

Commit cafe143

Browse files
HDVinnieHDVinnie
authored
Merge pull request #4699 from Roardom/tmdb-rate-limit
2 parents 1931a65 + 1962ef2 commit cafe143

File tree

4 files changed

+78
-16
lines changed

4 files changed

+78
-16
lines changed

‎app/Console/Commands/FetchMeta.php‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@
1717
namespace App\Console\Commands;
1818

1919
use App\Models\Torrent;
20+
use App\Jobs\ProcessIgdbGameJob;
21+
use App\Jobs\ProcessMovieJob;
22+
use App\Jobs\ProcessTvJob;
2023
use App\Services\Igdb\IgdbScraper;
2124
use App\Services\Tmdb\TMDBScraper;
2225
use Exception;
@@ -47,7 +50,7 @@ class FetchMeta extends Command
4750
final public function handle(): void
4851
{
4952
$start = now();
50-
$this->alert('Meta fetch queueing started. Fetching is done asynchronously in a separate job queue.');
53+
$this->alert('Meta fetch queueing started. Fetching is done synchronously within this command. This can take awhile (~1 work per second).');
5154

5255
$tmdbScraper = new TMDBScraper();
5356
$igdbScraper = new IgdbScraper();
@@ -61,11 +64,11 @@ final public function handle(): void
6164
->whereNotNull('tmdb_movie_id')
6265
->pluck('tmdb_movie_id');
6366

64-
$this->info('Queueing all tmdb movie metadata fetching');
67+
$this->info('Fetching '.$tmdbMovieIds->count().' movies');
6568

6669
foreach ($tmdbMovieIds as $id) {
67-
sleep(3);
68-
$tmdbScraper->movie($id);
70+
usleep(250_000);
71+
ProcessMovieJob::dispatchSync($id);
6972
$this->info("Movie metadata fetched for tmdb {$id}");
7073
}
7174

@@ -78,12 +81,12 @@ final public function handle(): void
7881
->whereNotNull('tmdb_tv_id')
7982
->pluck('tmdb_tv_id');
8083

81-
$this->info('Queueing all tmdb tv metadata fetching');
84+
$this->info('Fetching '.$tmdbTvIds->count().' tv series');
8285

8386
foreach ($tmdbTvIds as $id) {
84-
sleep(3);
85-
$tmdbScraper->tv($id);
86-
$this->info("Movie metadata fetched for tmdb {$id}");
87+
usleep(250_000);
88+
ProcessTvJob::dispatchSync($id);
89+
$this->info("TV metadata fetched for tmdb {$id}");
8790
}
8891

8992
$this->info('Querying all igdb game ids');
@@ -95,11 +98,11 @@ final public function handle(): void
9598
->whereNotNull('igdb')
9699
->pluck('igdb');
97100

98-
$this->info('Queueing all igdb game metadata fetching');
101+
$this->info('Fetching '.$igdbGameIds->count().' games');
99102

100103
foreach ($igdbGameIds as $id) {
101104
usleep(250_000);
102-
$igdbScraper->game($id);
105+
ProcessIgdbGameJob::dispatchSync($id);
103106
$this->info("Game metadata fetched for igdb {$id}");
104107
}
105108

‎app/Jobs/ProcessMovieJob.php‎

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,13 @@
2424
use App\Models\TmdbPerson;
2525
use App\Models\Torrent;
2626
use App\Services\Tmdb\Client;
27+
use DateTime;
2728
use Illuminate\Bus\Queueable;
2829
use Illuminate\Contracts\Queue\ShouldQueue;
2930
use Illuminate\Foundation\Bus\Dispatchable;
3031
use Illuminate\Queue\InteractsWithQueue;
32+
use Illuminate\Queue\Middleware\RateLimited;
33+
use Illuminate\Queue\Middleware\Skip;
3134
use Illuminate\Queue\Middleware\WithoutOverlapping;
3235
use Illuminate\Queue\SerializesModels;
3336

@@ -52,11 +55,27 @@ public function __construct(public int $id)
5255
*/
5356
public function middleware(): array
5457
{
55-
return [new WithoutOverlapping((string) $this->id)->dontRelease()->expireAfter(30)];
58+
return [
59+
Skip::when(cache()->has("tmdb-movie-scraper:{$this->id}")),
60+
new WithoutOverlapping((string) $this->id)->dontRelease()->expireAfter(30),
61+
new RateLimited('tmdb'),
62+
];
63+
}
64+
65+
/**
66+
* Determine the time at which the job should timeout.
67+
*/
68+
public function retryUntil(): DateTime
69+
{
70+
return now()->addDay();
5671
}
5772

5873
public function handle(): void
5974
{
75+
// TMDB caches their api responses for 8 hours, so don't abuse them
76+
77+
cache()->put("tmdb-movie-scraper:{$this->id}", now(), 8 * 3600);
78+
6079
// Movie
6180

6281
$movieScraper = new Client\Movie($this->id);
@@ -97,8 +116,18 @@ public function handle(): void
97116
$credits = $movieScraper->getCredits();
98117
$people = [];
99118

100-
foreach (array_unique(array_column($credits, 'tmdb_person_id')) as $person_id) {
101-
$people[] = (new Client\Person($person_id))->getPerson();
119+
foreach (array_unique(array_column($credits, 'tmdb_person_id')) as $personId) {
120+
// TMDB caches their api responses for 8 hours, so don't abuse them
121+
122+
$cacheKey = "tmdb-person-scraper:{$personId}";
123+
124+
if (cache()->has($cacheKey)) {
125+
continue;
126+
}
127+
128+
cache()->put($cacheKey, now(), 8 * 3600);
129+
130+
$people[] = (new Client\Person($personId))->getPerson();
102131
}
103132

104133
TmdbPerson::upsert($people, 'id');

‎app/Jobs/ProcessTvJob.php‎

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,13 @@
2424
use App\Models\Torrent;
2525
use App\Models\TmdbTv;
2626
use App\Services\Tmdb\Client;
27+
use DateTime;
2728
use Illuminate\Bus\Queueable;
2829
use Illuminate\Contracts\Queue\ShouldQueue;
2930
use Illuminate\Foundation\Bus\Dispatchable;
3031
use Illuminate\Queue\InteractsWithQueue;
32+
use Illuminate\Queue\Middleware\RateLimited;
33+
use Illuminate\Queue\Middleware\Skip;
3134
use Illuminate\Queue\Middleware\WithoutOverlapping;
3235
use Illuminate\Queue\SerializesModels;
3336

@@ -52,11 +55,27 @@ public function __construct(public int $id)
5255
*/
5356
public function middleware(): array
5457
{
55-
return [new WithoutOverlapping((string) $this->id)->dontRelease()->expireAfter(30)];
58+
return [
59+
Skip::when(cache()->has("tmdb-tv-scraper:{$this->id}")),
60+
new WithoutOverlapping((string) $this->id)->dontRelease()->expireAfter(30),
61+
new RateLimited('tmdb'),
62+
];
63+
}
64+
65+
/**
66+
* Determine the time at which the job should timeout.
67+
*/
68+
public function retryUntil(): DateTime
69+
{
70+
return now()->addDay();
5671
}
5772

5873
public function handle(): void
5974
{
75+
// TMDB caches their api responses for 8 hours, so don't abuse them
76+
77+
cache()->put("tmdb-tv-scraper:{$this->id}", now(), 8 * 3600);
78+
6079
// Tv
6180

6281
$tvScraper = new Client\TV($this->id);
@@ -99,8 +118,18 @@ public function handle(): void
99118
$credits = $tvScraper->getCredits();
100119
$people = [];
101120

102-
foreach (array_unique(array_column($credits, 'tmdb_person_id')) as $person_id) {
103-
$people[] = (new Client\Person($person_id))->getPerson();
121+
foreach (array_unique(array_column($credits, 'tmdb_person_id')) as $personId) {
122+
// TMDB caches their api responses for 8 hours, so don't abuse them
123+
124+
$cacheKey = "tmdb-person-scraper:{$personId}";
125+
126+
if (cache()->has($cacheKey)) {
127+
continue;
128+
}
129+
130+
cache()->put($cacheKey, now(), 8 * 3600);
131+
132+
$people[] = (new Client\Person($personId))->getPerson();
104133
}
105134

106135
TmdbPerson::upsert($people, 'id');

‎app/Providers/RouteServiceProvider.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ protected function configureRateLimiting(): void
8686
RateLimiter::for('rss', fn (Request $request) => Limit::perMinute(30)->by('rss'.$request->ip()));
8787
RateLimiter::for('authenticated-images', fn (Request $request): Limit => Limit::perMinute(200)->by('authenticated-images:'.$request->user()->id));
8888
RateLimiter::for('search', fn (Request $request): Limit => Limit::perMinute(100)->by('search:'.$request->user()->id));
89+
RateLimiter::for('tmdb', fn (): Limit => Limit::perSecond(2));
8990
}
9091

9192
protected function removeIndexPhpFromUrl(): void

0 commit comments

Comments
 (0)