Merge pull request #114 from DefangLabs/110-bullmq-bull-board-redis-sample

BullMQ Sample

Author: raphaeltm

samples/bullmq-bullboard-redis/.github/workflows/deploy.yaml

@@ -0,0 +1,31 @@
+name: deploy
+
+on:
+  push:
+    branches:
+      - main
+      - pr-test/*
+
+concurrency:
+  group: deploy
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+
+    steps:
+    - name: Checkout Repo
+      uses: actions/checkout@v4
+
+    - name: Install defang
+      run: . <(curl -Ls https://s.defang.io/install)
+
+    - name: Login to Defang
+      run: defang login
+
+    - name: Deploy
+      run: defang compose up -v --detach

samples/bullmq-bullboard-redis/README.md

@@ -0,0 +1,37 @@
+# BullMQ & BullBoard & Redis
+
+This sample project demonstrates how to deploy a BullMQ message queue on top of managed Redis with a queue processor and a dashboard to monitor the queue.
+
+Once your app is up and running you can go to the `/board` route for the `board` service to see the Bull Board dashboard and use the username `admin` and the password you set to log in (see [Deploying](#deploying)).
+
+To add a job to the queue, you can go to the `/add` route of the `api` service. This will use some default values so you can test things out. You can also see an example of a post request in the [sample http request](./api/add.test.http) file.
+
+The `worker` service is the queue processor that will process the jobs added to the queue. You can see in the `compose.yaml` file that the `worker` service is set to scale to 2 instances. This means that there will be 2 workers processing jobs from the queue. You can set this to your desired number of workers, but we wanted to show how you can increase the number of workers to handle more jobs.
+
+
+## Prerequisites
+
+1. Download <a href="https://github.com/defang-io/defang">Defang CLI</a>
+2. (optional) If you are using <a href="https://docs.defang.io/docs/concepts/defang-byoc">Defang BYOC</a>, make sure you have properly <a href="https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html">authenticated your AWS account</a>.
+
+## Deploying
+
+1. Open the terminal and type `defang login`
+2. Run `defang config set BOARD_PASSWORD` to set the password for the BullBoard dashboard.
+3. Type `defang compose up` in the CLI.
+4. Your app will be running within a few minutes.
+
+
+## Local Development
+
+1. Run `docker compose -f compose.dev.yaml up`
+
+---
+
+Title: BullMQ & BullBoard & Redis
+
+Short Description: A sample project with BullMQ, BullBoard, and Redis.
+
+Tags: BullMQ, BullBoard, Redis, Express, Node.js, Message Queue
+
+Languages: nodejs, javascript

samples/bullmq-bullboard-redis/api/.dockerignore

@@ -0,0 +1,82 @@
+# Logs
+logs
+*.log
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# TypeScript v1.x
+*.ts~
+*.tsx~
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+.env.test
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+
+# next.js build output
+.next
+
+# nuxt.js build / generate output
+.nuxt
+dist/
+
+# vuepress build output
+.vuepress/dist
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/

samples/bullmq-bullboard-redis/api/.gitignore

@@ -0,0 +1,88 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# TypeScript v1.x
+*.ts~
+*.tsx~
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+.env.test
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+
+# next.js build output
+.next
+
+# nuxt.js build / generate output
+.nuxt
+dist/
+
+# vuepress build output
+.vuepress/dist
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/

samples/bullmq-bullboard-redis/api/Dockerfile

@@ -0,0 +1,17 @@
+# Use the official Node.js 14 LTS image as the base image
+FROM node:20-alpine
+
+# Set the working directory
+WORKDIR /app
+
+# Copy package.json and package-lock.json
+COPY package*.json ./
+
+# Install dependencies
+RUN npm ci
+
+# Copy the rest of the application source code
+COPY . .
+
+# Define the command to run the application
+CMD ["node", "index.mjs"]

samples/bullmq-bullboard-redis/api/add.test.http

@@ -0,0 +1,10 @@
+POST http://localhost:3001/add HTTP/1.1
+Content-Type: application/json
+
+{
+    "name": "process-object",
+    "data": {
+        "id": 1,
+        "name": "object-to-process"
+    }
+}

samples/bullmq-bullboard-redis/api/index.mjs

@@ -0,0 +1,39 @@
+import { Queue, Worker } from 'bullmq';
+import IORedis from 'ioredis';
+import express from 'express';
+
+const connectionString = process.env.REDIS;
+const connection = new IORedis(connectionString);
+
+const myQueue = new Queue(process.env.QUEUE, { connection });
+
+const app = express();
+app.use(express.json());
+
+// For health check
+app.get('/', async (req, res) => {
+    res.send('All good');
+});
+
+// GET example
+app.get('/add', async (req, res) => {
+    const params = req.query;
+    const name = params.name || 'myjob';
+    const data = JSON.parse(params.data || '{}');
+    
+    await myQueue.add(name, data);
+    res.send('Job added');
+});
+
+// POST example
+app.post('/add', async (req, res) => {
+    const body = req.body;
+    const name = body.name || 'myjob';
+    const data = body.data || {};
+
+    await myQueue.add(name, data);
+});
+
+app.listen(3001, () => {
+    console.log('Server is running on http://localhost:3001');
+});