From 45254e6d42b1d6fa620e2e4d108b4f7667c933fa Mon Sep 17 00:00:00 2001 From: D-K-P <8297864+D-K-P@users.noreply.github.com> Date: Fri, 11 Oct 2024 11:18:46 +0100 Subject: [PATCH 1/3] Added prerequisites to ffmpeg and puppeteer --- .../guides/examples/ffmpeg-video-processing.mdx | 7 ++++++- docs/guides/examples/puppeteer.mdx | 17 +++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/guides/examples/ffmpeg-video-processing.mdx b/docs/guides/examples/ffmpeg-video-processing.mdx index 39687e3ab5..229c9c0907 100644 --- a/docs/guides/examples/ffmpeg-video-processing.mdx +++ b/docs/guides/examples/ffmpeg-video-processing.mdx @@ -6,7 +6,12 @@ description: "These examples show you how to process videos in various ways usin import LocalDevelopment from "/snippets/local-development-extensions.mdx"; -## Adding the FFmpeg build extension +## Prerequisites + +- A project with [Trigger.dev initialized](/quick-start) +- [FFmpeg](https://www.ffmpeg.org/download.html) installed on your machine + +### Adding the FFmpeg build extension To use these example tasks, you'll first need to add our FFmpeg extension to your project configuration like this: diff --git a/docs/guides/examples/puppeteer.mdx b/docs/guides/examples/puppeteer.mdx index 9ec41ea610..b51827a82d 100644 --- a/docs/guides/examples/puppeteer.mdx +++ b/docs/guides/examples/puppeteer.mdx @@ -7,6 +7,11 @@ description: "These examples demonstrate how to use Puppeteer with Trigger.dev." import LocalDevelopment from "/snippets/local-development-extensions.mdx"; import ScrapingWarning from "/snippets/web-scraping-warning.mdx"; +## Prerequisites + +- A project with [Trigger.dev initialized](/quick-start) +- [Puppeteer](https://pptr.dev/guides/installation) installed on your machine + ## Overview There are 3 example tasks to follow on this page: @@ -15,7 +20,7 @@ There are 3 example tasks to follow on this page: 2. [Generate a PDF from a web page](/guides/examples/puppeteer#generate-a-pdf-from-a-web-page) 3. [Scrape content from a web page](/guides/examples/puppeteer#scrape-content-from-a-web-page) - + ## Build configurations @@ -133,7 +138,6 @@ export const puppeteerWebpageToPDF = task({ return { pdfUrl: s3Url }; }, }); - ``` ### Testing your task @@ -146,9 +150,10 @@ There's no payload required for this task so you can just click "Run test" from In this example we use [Puppeteer](https://pptr.dev/) with a [BrowserBase](https://www.browserbase.com/) proxy to scrape the GitHub stars count from the [Trigger.dev](https://trigger.dev) landing page and log it out. See [this list](/guides/examples/puppeteer#proxying) for more proxying services we recommend. - - When web scraping, you MUST use the technique below which uses a proxy with Puppeteer. Direct scraping without using `browserWSEndpoint` is prohibited and will result in account suspension. - + + When web scraping, you MUST use the technique below which uses a proxy with Puppeteer. Direct + scraping without using `browserWSEndpoint` is prohibited and will result in account suspension. + ### Task code @@ -209,4 +214,4 @@ Here are a list of proxy services we recommend: - [Browserless](https://browserless.io/) - [Oxylabs](https://oxylabs.io/) - [ScrapingBee](https://scrapingbee.com/) -- [Smartproxy](https://smartproxy.com/) \ No newline at end of file +- [Smartproxy](https://smartproxy.com/) From f4dfde17694f697ab488da2f0dcbbca543a129a3 Mon Sep 17 00:00:00 2001 From: D-K-P <8297864+D-K-P@users.noreply.github.com> Date: Fri, 11 Oct 2024 11:30:19 +0100 Subject: [PATCH 2/3] Improved the sharp docs --- .../examples/sharp-image-processing.mdx | 52 +++++++++++-------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/docs/guides/examples/sharp-image-processing.mdx b/docs/guides/examples/sharp-image-processing.mdx index 5259eff988..ac936dbb45 100644 --- a/docs/guides/examples/sharp-image-processing.mdx +++ b/docs/guides/examples/sharp-image-processing.mdx @@ -8,9 +8,15 @@ import LocalDevelopment from "/snippets/local-development-extensions.mdx"; ## Overview -This task optimizes and watermarks an image using the Sharp library, and then uploads the processed image to R2 storage. +This task processes and watermarks an image using the Sharp library, and then uploads it to R2 storage. -## Adding build configurations +## Prerequisites + +- A project with [Trigger.dev initialized](/quick-start) +- The [Sharp](https://sharp.pixelplumbing.com/install) library installed on your machine +- An R2-compatible object storage service, such as [Cloudflare R2](https://developers.cloudflare.com/r2) + +## Adding the build configuration To use this example, you'll first need to add these build settings to your `trigger.config.ts` file: @@ -34,9 +40,9 @@ export default defineConfig({ ## Key features - - Resizes and rotates an image - - Adds a watermark to the image - - Uploads the processed image to R2 storage +- Resizes a JPEG image to 800x800 pixels +- Adds a watermark to the image, positioned in the bottom-right corner, using a PNG image +- Uploads the processed image to R2 storage ## Task code @@ -44,12 +50,11 @@ export default defineConfig({ import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3"; import { logger, task } from "@trigger.dev/sdk/v3"; import fs from "fs/promises"; -import fetch from "node-fetch"; import os from "os"; import path from "path"; import sharp from "sharp"; -// Initialize R2 client +// Initialize R2 client using your R2 account details const r2Client = new S3Client({ region: "auto", endpoint: process.env.R2_ENDPOINT, @@ -61,6 +66,10 @@ const r2Client = new S3Client({ export const sharpProcessImage = task({ id: "sharp-process-image", + // Only retry this task once if it fails + retry: { + maxAttempts: 1, + }, run: async (payload: { imageUrl: string; watermarkUrl: string }) => { const { imageUrl, watermarkUrl } = payload; @@ -76,10 +85,9 @@ export const sharpProcessImage = task({ const imageBuffer = await imageResponse.arrayBuffer(); const watermarkBuffer = await watermarkResponse.arrayBuffer(); - // Optimize the image using Sharp + // Process the image using Sharp await sharp(Buffer.from(imageBuffer)) - .rotate(90) // Rotate the image by 90 degrees - .resize(800, 600) // Resize the image to 800x600 + .resize(800, 800) // Resize the image to 800x800 .composite([ { input: Buffer.from(watermarkBuffer), @@ -89,34 +97,32 @@ export const sharpProcessImage = task({ .toFormat("jpeg") .toFile(outputPath); - // Log the output file path - logger.log(`Optimized image saved at: ${outputPath}`); + logger.log(`Image processed.`); - // Read the optimized image file - const optimizedImageBuffer = await fs.readFile(outputPath); + // Read the processed image file + const processedImageBuffer = await fs.readFile(outputPath); - // Upload the optimized image to R2, replacing slashes with underscores + // Upload the image to R2, replacing slashes with underscores const r2Key = `processed-images/${path.basename(outputPath)}`; const uploadParams = { Bucket: process.env.R2_BUCKET, Key: r2Key, - Body: optimizedImageBuffer, + Body: processedImageBuffer, }; // Upload the image to R2 and get the URL await r2Client.send(new PutObjectCommand(uploadParams)); - const r2Url = `https://${process.env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com/${process.env.R2_BUCKET}/${r2Key}`; - logger.log("Optimized image uploaded to R2", { url: r2Url }); + logger.log("Processed image uploaded to R2", { + path: `/${process.env.R2_BUCKET}/${r2Key}`, + }); // Delete the temporary file await fs.unlink(outputPath); + logger.log("Temporary file deleted", { outputPath: outputPath }); - // Return the optimized image buffer, file path, and R2 URL return { - optimizedImageBuffer, - optimizedImagePath: outputPath, - r2Url, + message: `New image uploaded to /${process.env.R2_BUCKET}/${r2Key}`, }; }, }); @@ -133,4 +139,4 @@ To test this task in the dashboard, you can use the following payload: } ``` - \ No newline at end of file + From 445fda7f0969afa3ae9193968fd9304a95bb5aeb Mon Sep 17 00:00:00 2001 From: D-K-P <8297864+D-K-P@users.noreply.github.com> Date: Fri, 11 Oct 2024 16:53:42 +0100 Subject: [PATCH 3/3] Updated sharp task code --- .../examples/sharp-image-processing.mdx | 72 ++++++++----------- 1 file changed, 30 insertions(+), 42 deletions(-) diff --git a/docs/guides/examples/sharp-image-processing.mdx b/docs/guides/examples/sharp-image-processing.mdx index ac936dbb45..becfc39c3c 100644 --- a/docs/guides/examples/sharp-image-processing.mdx +++ b/docs/guides/examples/sharp-image-processing.mdx @@ -47,7 +47,8 @@ export default defineConfig({ ## Task code ```ts trigger/sharp-image-processing.ts -import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3"; +import { S3Client } from "@aws-sdk/client-s3"; +import { Upload } from "@aws-sdk/lib-storage"; import { logger, task } from "@trigger.dev/sdk/v3"; import fs from "fs/promises"; import os from "os"; @@ -66,18 +67,11 @@ const r2Client = new S3Client({ export const sharpProcessImage = task({ id: "sharp-process-image", - // Only retry this task once if it fails - retry: { - maxAttempts: 1, - }, + retry: { maxAttempts: 1 }, run: async (payload: { imageUrl: string; watermarkUrl: string }) => { const { imageUrl, watermarkUrl } = payload; + const outputPath = path.join(os.tmpdir(), `output_${Date.now()}.jpg`); - // Generate temporary and output file names - const tempDirectory = os.tmpdir(); - const outputPath = path.join(tempDirectory, `output_${Date.now()}.jpg`); - - // Fetch the image and watermark const [imageResponse, watermarkResponse] = await Promise.all([ fetch(imageUrl), fetch(watermarkUrl), @@ -85,45 +79,39 @@ export const sharpProcessImage = task({ const imageBuffer = await imageResponse.arrayBuffer(); const watermarkBuffer = await watermarkResponse.arrayBuffer(); - // Process the image using Sharp await sharp(Buffer.from(imageBuffer)) - .resize(800, 800) // Resize the image to 800x800 + .resize(800, 800) // Resize the image to 800x800px .composite([ { input: Buffer.from(watermarkBuffer), gravity: "southeast", // Position the watermark in the bottom-right corner }, ]) - .toFormat("jpeg") - .toFile(outputPath); - - logger.log(`Image processed.`); - - // Read the processed image file - const processedImageBuffer = await fs.readFile(outputPath); - - // Upload the image to R2, replacing slashes with underscores - const r2Key = `processed-images/${path.basename(outputPath)}`; - - const uploadParams = { - Bucket: process.env.R2_BUCKET, - Key: r2Key, - Body: processedImageBuffer, - }; - - // Upload the image to R2 and get the URL - await r2Client.send(new PutObjectCommand(uploadParams)); - logger.log("Processed image uploaded to R2", { - path: `/${process.env.R2_BUCKET}/${r2Key}`, - }); - - // Delete the temporary file - await fs.unlink(outputPath); - logger.log("Temporary file deleted", { outputPath: outputPath }); - - return { - message: `New image uploaded to /${process.env.R2_BUCKET}/${r2Key}`, - }; + .jpeg() // Convert to jpeg + .toBuffer() // Convert to buffer + .then(async (outputBuffer) => { + await fs.writeFile(outputPath, outputBuffer); // Write the buffer to file + + const r2Key = `processed-images/${path.basename(outputPath)}`; + const uploadParams = { + Bucket: process.env.R2_BUCKET, + Key: r2Key, + Body: await fs.readFile(outputPath), + }; + + const upload = new Upload({ + client: r2Client, + params: uploadParams, + }); + + await upload.done(); + logger.log("Image uploaded to R2 storage.", { + path: `/${process.env.R2_BUCKET}/${r2Key}`, + }); + + await fs.unlink(outputPath); // Clean up the temporary file + return { r2Key }; + }); }, }); ```