diff --git a/.env.railway b/.env.railway new file mode 100644 index 0000000..d4defd9 --- /dev/null +++ b/.env.railway @@ -0,0 +1,5 @@ +REDIS_URL="${{Redis (Webhook).REDIS_URL}}" # Auto-configured Redis for webhook event storage +PORT="3000" # Server port for webhook endpoints +NODE_ENV="production" # Environment mode (keep as production for Railway) +TARGET_PLATFORM="" # Platform integration: "discord" or "telegram" - REQUIRED +UNTHREAD_WEBHOOK_SECRET="" # Webhook secret from Unthread dashboard - REQUIRED diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 91aa5d8..9b32b8c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,7 +52,7 @@ jobs: ${{ env.REGISTRY_GHCR }}:dev-${{ steps.meta.outputs.short_sha }} labels: | org.opencontainers.image.title=Unthread Webhook Server - org.opencontainers.image.description=Development build of Unthread Webhook Server + org.opencontainers.image.description=A reliable, production-ready Node.js server for processing Unthread.io webhooks with signature verification and smart platform handling. org.opencontainers.image.version=dev-${{ steps.meta.outputs.short_sha }} org.opencontainers.image.created=${{ steps.meta.outputs.build_date }} org.opencontainers.image.revision=${{ github.sha }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 810fc21..f99ac79 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -76,8 +76,7 @@ jobs: # Combine all tags ALL_TAGS="$DOCKERHUB_TAGS,$GHCR_TAGS" - - echo "tags=$ALL_TAGS" >> $GITHUB_OUTPUT + echo "tags=$ALL_TAGS" >> $GITHUB_OUTPUT - name: Build and push production images uses: docker/build-push-action@v5 @@ -88,14 +87,14 @@ jobs: tags: ${{ steps.tags.outputs.tags }} labels: | org.opencontainers.image.title=Unthread Webhook Server - org.opencontainers.image.description=A Node.js server application that receives webhook events from Unthread.io + org.opencontainers.image.description=A reliable, production-ready Node.js server for processing Unthread.io webhooks with signature verification and smart platform handling. org.opencontainers.image.version=${{ steps.version.outputs.version }} org.opencontainers.image.created=${{ steps.version.outputs.build_date }} org.opencontainers.image.revision=${{ github.sha }} org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} org.opencontainers.image.url=${{ github.server_url }}/${{ github.repository }} org.opencontainers.image.licenses=GPL-3.0 - cache-from: type=gha + cache-from: type=gha cache-to: type=gha,mode=max - name: Run Trivy vulnerability scanner diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f751b94..d4ba62c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,7 @@ Any contributions are welcome, encouraged, and valued. See the following informa ## ๐Ÿ“‹ Code of Conduct -This project and everyone participating in it is governed by the project's Code of Conduct. By participating, you are expected to uphold this code. Please report unacceptable behavior to . +This project and everyone participating in it is governed by the project's [Code of Conduct](./CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to . ## ๐Ÿ’– How to Contribute @@ -14,7 +14,7 @@ There are many ways to contribute to this open source project. Any contributions If you can write code then create a pull request to this repo and I will review your code. Please consider submitting your pull request to the `dev` branch. I will auto reject if you submit your pull request to the `main` branch. -#### ๐Ÿ”ง Setup +#### ๐Ÿ”ง Development Setup To get started with development: @@ -44,6 +44,10 @@ To get started with development: brew services start redis # macOS sudo systemctl start redis-server # Linux docker run -d -p 6379:6379 redis:alpine # Docker + + # OR for full Docker setup with proper naming: + docker network create unthread-integration-network + docker-compose up -d redis-webhook ``` 5. **Start the project in development mode** @@ -97,11 +101,13 @@ src/ #### ๐ŸŽฏ Development Guidelines - **TypeScript First**: All code must be written in TypeScript with strict type checking +- **Structured Logging**: Use `@wgtechlabs/log-engine` for all logging with built-in PII protection and security features - **Error Handling**: Implement comprehensive error handling with detailed logging - **Package Manager**: Use Yarn exclusively (enforced via preinstall script) - **Code Style**: Follow existing patterns and maintain consistency - **Environment**: Use Node.js 20+ for development - **Redis Integration**: Ensure Redis connectivity for all webhook-related features +- **Webhook Integration**: Ensure compatibility with [`wgtechlabs/unthread-telegram-bot`](https://github.com/wgtechlabs/unthread-telegram-bot) #### ๐Ÿงช Testing Guidelines @@ -153,11 +159,151 @@ For other bugs, please create an issue with: ### ๐Ÿ’ก Feature Requests We welcome suggestions for new features! Please create an issue with: + - Clear description of the feature - Use case and benefits - Any implementation considerations - Examples or mockups if applicable +## ๐Ÿ“Š Advanced Logging Security with Log Engine + +This project uses [`@wgtechlabs/log-engine`](https://github.com/wgtechlabs/log-engine) for enterprise-grade logging with built-in security features and comprehensive PII protection. + +### ๐Ÿ”’ **Automatic Security Features** + +**Zero Configuration PII Protection:** + +- **Automatic Redaction**: Passwords, tokens, emails, API keys, and 50+ sensitive patterns are automatically protected +- **Deep Object Scanning**: Recursively scans nested objects and arrays for sensitive data +- **Content Truncation**: Large payloads are automatically truncated to prevent log bloat +- **Environment-Based Control**: Security automatically adapts based on NODE_ENV settings + +**Built-in Patterns Protected:** + +- **Authentication**: `password`, `token`, `apiKey`, `secret`, `jwt`, `auth`, `sessionId` +- **Personal Info**: `email`, `phone`, `ssn`, `firstName`, `lastName`, `address` +- **Financial**: `creditCard`, `cvv`, `bankAccount`, `routingNumber` +- **System**: `clientSecret`, `privateKey`, `webhookSecret`, `unthreadSecret` + +### ๐Ÿ›ก๏ธ **Advanced Security Configuration** + +**Custom Enterprise Protection:** + +```javascript +import { LogEngine } from '@wgtechlabs/log-engine'; + +// Add custom patterns for enterprise-specific data +LogEngine.addCustomRedactionPatterns([ + /internal.*/i, // Matches any field starting with "internal" + /company.*/i, // Matches any field starting with "company" + /webhook.*/i, // Matches webhook-specific fields + /unthread.*/i // Matches unthread-specific fields +]); + +// Add dynamic sensitive field names +LogEngine.addSensitiveFields([ + 'webhookSecret', + 'unthreadWebhookSecret', + 'unthreadApiKey', + 'redisPassword' +]); +``` + +**Secure Logging Examples:** + +```javascript +// โœ… Automatic protection - no configuration needed +LogEngine.info('Webhook authentication', { + webhookId: '123456789', // โœ… Visible + webhookSecret: 'secret123', // โŒ [REDACTED] + targetPlatform: 'telegram', // โœ… Visible + unthreadApiKey: 'key_123' // โŒ [REDACTED] +}); + +// โœ… Event processing protection +LogEngine.info('Event processing', { + eventType: 'message_created', // โœ… Visible + eventId: 'evt_001', // โœ… Visible + signature: 'sha256=...', // โŒ [REDACTED] + payload: { /* large data */ } // Automatically truncated +}); + +// โœ… Redis queue security +LogEngine.info('Queue publishing', { + queueName: 'unthread-events', // โœ… Visible + platform: 'unthread', // โœ… Visible + redisUrl: 'redis://localhost', // โŒ [REDACTED] + eventCount: 5 // โœ… Visible +}); +``` + +### โš™๏ธ **Environment Configuration** + +**Production Security (Recommended):** + +```bash +NODE_ENV=production # Full PII protection enabled +LOG_REDACTION_TEXT="[SECURE]" # Custom redaction text +LOG_MAX_CONTENT_LENGTH=150 # Truncate large content +``` + +**Development Debugging:** + +```bash +NODE_ENV=development # Redaction disabled for debugging +LOG_REDACTION_DISABLED=true # Explicit disable +DEBUG_FULL_PAYLOADS=true # Show complete data +``` + +**Custom Security Configuration:** + +```bash +# Custom sensitive fields (comma-separated) +LOG_SENSITIVE_FIELDS="webhookSecret,unthreadSecret,redisPassword" + +# Custom redaction patterns (JSON array) +LOG_CUSTOM_PATTERNS='["/internal.*/i", "/company.*/i"]' + +# Truncation settings +LOG_MAX_CONTENT_LENGTH=200 +LOG_TRUNCATION_TEXT="... [CONFIDENTIAL_TRUNCATED]" +``` + +### ๐Ÿ”ง **Development & Debugging** + +**Raw Logging for Development:** + +```javascript +// โš ๏ธ Use with caution - bypasses all redaction +LogEngine.debugRaw('Full webhook payload', { + password: 'visible', // โš ๏ธ Visible (not redacted) + apiKey: 'full-key-visible' // โš ๏ธ Visible (not redacted) +}); + +// Temporary redaction bypass +LogEngine.withoutRedaction().info('Debug mode', sensitiveData); + +// Test field redaction +const isRedacted = LogEngine.testFieldRedaction('webhookSecret'); // true +const currentConfig = LogEngine.getRedactionConfig(); +``` + +### ๐Ÿ“Š **Logging Benefits for This Webhook Server** + +**Security Compliance:** + +- **GDPR Ready**: Automatic PII protection for European compliance +- **Data Minimization**: Only necessary data is logged +- **Audit Trails**: Complete security event logging with timestamps +- **Incident Response**: Quick identification of security events + +**Operational Benefits:** + +- **Color-Coded Output**: Easy visual identification of log levels (๐Ÿ”ต INFO, ๐ŸŸก WARN, ๐Ÿ”ด ERROR) +- **Structured Logging**: Consistent format across all webhook components +- **Performance Optimized**: Minimal overhead with intelligent processing +- **TypeScript Support**: Full type safety and IDE integration + --- ๐Ÿ’ป with โค๏ธ by [Waren Gonzaga](https://warengonzaga.com), [WG Technology Labs](https://wgtechlabs.com), and [Him](https://www.youtube.com/watch?v=HHrxS4diLew&t=44s) ๐Ÿ™ diff --git a/Dockerfile b/Dockerfile index f13570a..280b351 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,51 +1,93 @@ -# Multi-stage build for Unthread Webhook Server - -# Build stage -FROM node:20-alpine AS builder - -# Set working directory -WORKDIR /app - -# Copy package manager files -COPY package.json yarn.lock .yarnrc ./ +# ============================================================================= +# UNTHREAD WEBHOOK SERVER - DOCKERFILE +# ============================================================================= +# Multi-stage Docker build for the Unthread Webhook Server +# +# Build stages: +# 1. base - Base Alpine image with Node.js and security updates +# 2. deps - Install production dependencies only +# 3. build - Install dev dependencies and build the application +# 4. final - Create minimal runtime image with built app +# +# Usage: +# docker build -t unthread-webhook-server . +# docker run --env-file .env unthread-webhook-server +# +# ============================================================================= + +# syntax=docker/dockerfile:1 + +# Use Node.js 22.16 LTS Alpine with security patches +ARG NODE_VERSION=22.16-alpine3.21 + +# ============================================================================= +# STAGE 1: Base Image +# ============================================================================= +# Alpine Linux 3.21 base for minimal image size with latest security updates +FROM node:${NODE_VERSION} AS base + +# Install security updates for Alpine packages +RUN apk update && apk upgrade && \ + apk add --no-cache dumb-init && \ + rm -rf /var/cache/apk/* + +# Set working directory for all subsequent stages +WORKDIR /usr/src/app + +# ============================================================================= +# STAGE 2: Production Dependencies +# ============================================================================= +# Install only production dependencies for runtime +FROM base AS deps + +# Use bind mounts and cache for faster builds +# Downloads dependencies without copying package files into the layer +RUN --mount=type=bind,source=package.json,target=package.json \ + --mount=type=bind,source=yarn.lock,target=yarn.lock \ + --mount=type=cache,target=/root/.yarn \ + yarn install --production --frozen-lockfile + +# ============================================================================= +# STAGE 3: Build Application +# ============================================================================= +# Install dev dependencies and build the TypeScript application +FROM deps AS build # Install all dependencies (including devDependencies for building) -RUN yarn install --frozen-lockfile --ignore-scripts - -# Copy source code and build configuration -COPY src/ ./src/ -COPY tsconfig.json ./ - -# Build the TypeScript application -RUN yarn build - -# Production stage -FROM node:20-alpine AS production - -# Set working directory -WORKDIR /app - -# Copy package manager files -COPY package.json yarn.lock .yarnrc ./ +RUN --mount=type=bind,source=package.json,target=package.json \ + --mount=type=bind,source=yarn.lock,target=yarn.lock \ + --mount=type=cache,target=/root/.yarn \ + yarn install --frozen-lockfile + +# Copy source code and build the application +COPY . . +RUN yarn run build + +# ============================================================================= +# STAGE 4: Final Runtime Image +# ============================================================================= +# Minimal production image with only necessary files +FROM base AS final + +# Set production environment with security options +ENV NODE_ENV=production \ + NODE_OPTIONS="--enable-source-maps --max-old-space-size=512" + +# Create a dedicated user for the application +RUN addgroup -g 1001 -S nodejs && \ + adduser -S nodejs -u 1001 -G nodejs -# Set environment to production -ENV NODE_ENV=production -# Install only production dependencies -RUN yarn install --frozen-lockfile --production --ignore-scripts && \ - yarn cache clean +# Copy package.json for package manager commands +COPY --chown=nodejs:nodejs package.json . -# Copy built application from builder stage -COPY --from=builder /app/dist ./dist +# Copy production dependencies and built application +COPY --from=deps --chown=nodejs:nodejs /usr/src/app/node_modules ./node_modules +COPY --from=build --chown=nodejs:nodejs /usr/src/app/dist ./dist # Copy environment example (for reference) -COPY .env.example ./ - -# Create non-root user for security -RUN addgroup -g 1001 -S nodejs && \ - adduser -S nodejs -u 1001 +COPY --chown=nodejs:nodejs .env.example ./ -# Change ownership of the app directory to nodejs user -RUN chown -R nodejs:nodejs /app +# Switch to non-root user USER nodejs # Expose the port the app runs on @@ -55,5 +97,6 @@ EXPOSE 3000 HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ CMD node -e "require('http').get('http://localhost:3000/health', (res) => { process.exit(res.statusCode === 200 ? 0 : 1) }).on('error', () => process.exit(1))" -# Start the application +# Use dumb-init for proper signal handling and start the application +ENTRYPOINT ["dumb-init", "--"] CMD ["node", "dist/app.js"] \ No newline at end of file diff --git a/README.md b/README.md index 1a5077f..be8d893 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Unthread Webhook Server ๐ŸŽซโšก [![made by](https://img.shields.io/badge/made%20by-WG%20Tech%20Labs-0060a0.svg?logo=github&longCache=true&labelColor=181717&style=flat-square)](https://github.com/wgtechlabs) -[![node](https://img.shields.io/badge/node-%3E%3D20-green.svg?style=flat-square&labelColor=181717&logo=node.js&logoColor=white)](https://nodejs.org/) [![typescript](https://img.shields.io/badge/typescript-5.x-blue.svg?style=flat-square&labelColor=181717&logo=typescript&logoColor=white)](https://www.typescriptlang.org/) [![sponsors](https://img.shields.io/badge/sponsor-%E2%9D%A4-%23db61a2.svg?&logo=github&logoColor=white&labelColor=181717&style=flat-square)](https://github.com/sponsors/wgtechlabs) [![release](https://img.shields.io/github/release/wgtechlabs/unthread-webhook-server.svg?logo=github&labelColor=181717&color=green&style=flat-square)](https://github.com/wgtechlabs/unthread-webhook-server/releases) [![star](https://img.shields.io/github/stars/wgtechlabs/unthread-webhook-server.svg?&logo=github&labelColor=181717&color=yellow&style=flat-square)](https://github.com/wgtechlabs/unthread-webhook-server/stargazers) [![license](https://img.shields.io/github/license/wgtechlabs/unthread-webhook-server.svg?&logo=github&labelColor=181717&style=flat-square)](https://github.com/wgtechlabs/unthread-webhook-server/blob/main/license) +[![release workflow](https://img.shields.io/github/actions/workflow/status/wgtechlabs/unthread-webhook-server/release.yml?branch=main&style=flat-square&logo=github&labelColor=181717&label=release)](https://github.com/wgtechlabs/unthread-webhook-server/actions/workflows/release.yml) [![build workflow](https://img.shields.io/github/actions/workflow/status/wgtechlabs/unthread-webhook-server/build.yml?branch=dev&style=flat-square&logo=github&labelColor=181717&label=build)](https://github.com/wgtechlabs/unthread-webhook-server/actions/workflows/build.yml) [![node](https://img.shields.io/badge/node-%3E%3D20-green.svg?style=flat-square&labelColor=181717&logo=node.js&logoColor=white)](https://nodejs.org/) [![typescript](https://img.shields.io/badge/typescript-5.x-blue.svg?style=flat-square&labelColor=181717&logo=typescript&logoColor=white)](https://www.typescriptlang.org/) [![sponsors](https://img.shields.io/badge/sponsor-%E2%9D%A4-%23db61a2.svg?&logo=github&logoColor=white&labelColor=181717&style=flat-square)](https://github.com/sponsors/wgtechlabs) [![version](https://img.shields.io/github/release/wgtechlabs/unthread-webhook-server.svg?logo=github&labelColor=181717&color=green&style=flat-square&label=version)](https://github.com/wgtechlabs/unthread-webhook-server/releases) [![star](https://img.shields.io/github/stars/wgtechlabs/unthread-webhook-server.svg?&logo=github&labelColor=181717&color=yellow&style=flat-square)](https://github.com/wgtechlabs/unthread-webhook-server/stargazers) [![license](https://img.shields.io/github/license/wgtechlabs/unthread-webhook-server.svg?&logo=github&labelColor=181717&style=flat-square)](https://github.com/wgtechlabs/unthread-webhook-server/blob/main/license) -A **production-ready Node.js webhook server** for Unthread.io integration. Built with TypeScript, Express.js, and Redis for reliable webhook event processing with HMAC signature verification and intelligent platform detection. +A reliable, production-ready Node.js server for processing Unthread.io webhooks with signature verification and smart platform handling. Built with TypeScript, Express.js, and Redis, this webhook server provides secure HMAC-SHA256 signature validation, intelligent event deduplication, and seamless integration with multiple platforms including Discord and Telegram. The server automatically detects event sources, processes various webhook events (conversations, messages, status updates), and efficiently queues them through Redis for downstream consumption by your bot applications, ensuring reliable and scalable webhook processing for your Unthread.io integrations. ## ๐Ÿค— Special Thanks @@ -50,30 +50,41 @@ Server runs on `http://localhost:3000` with endpoints: - `GET /health` - Health check - `POST /unthread-webhook` - Webhook endpoint +## ๐Ÿš‚ One-Click Deploy + +Deploy instantly to Railway with a single click: + +[![deploy on railway](https://railway.com/button.svg)](https://railway.com/deploy/unthread-webhook-ser?referralCode=dTwT-i) + ## ๐Ÿณ Docker Setup ```bash -# 1. Copy environment template +# 1. Create external network (if not already created) +docker network create unthread-integration-network + +# 2. Copy environment template cp .env.example .env # Edit .env with your webhook secret -# 2. Start with Docker Compose +# 3. Start with Docker Compose docker-compose up -d -# 3. Check status +# 4. Check status docker-compose ps -# 4. View logs -docker-compose logs -f +# 5. View logs +docker-compose logs -f webhook-server +docker-compose logs -f redis-webhook -# 5. Stop services +# 6. Stop services docker-compose down ``` **Environment Files:** - `.env` - Single config file for both local development and Docker -- `.env.example` - Template (Redis URL gets overridden automatically for Docker) +- `.env.example` - Template with default values +- `.env.railway` - Railway deployment template ## โš™๏ธ Configuration diff --git a/docker-compose.yml b/docker-compose.yml index 90ec322..c76ad83 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,12 +1,36 @@ +# ============================================================================= +# UNTHREAD WEBHOOK SERVER - DOCKER COMPOSE CONFIGURATION +# ============================================================================= +# Complete application stack for the Unthread Webhook Server +# +# Services: +# โ”Œโ”€ webhook-server โ†’ Main webhook processing application +# โ””โ”€ redis-webhook โ†’ Redis for webhook message queuing +# +# Usage: +# docker-compose up -d # Start all services +# docker-compose logs -f webhook-server # View webhook server logs +# docker-compose exec redis-webhook redis-cli +# +# Prerequisites: +# - Copy .env.example to .env and configure your tokens +# - Ensure external network exists: docker network create unthread-integration-network +# +# ============================================================================= + version: '3.8' services: - redis: + # ============================================================================= + # REDIS - WEBHOOK OPERATIONS + # ============================================================================= + # Dedicated Redis instance for webhook message queuing and communication + redis-webhook: image: redis:7-alpine ports: - "6379:6379" volumes: - - redis_data:/data + - redis_webhook_data:/data restart: unless-stopped healthcheck: test: ["CMD", "redis-cli", "ping"] @@ -14,6 +38,13 @@ services: timeout: 3s retries: 3 start_period: 30s + networks: + - unthread-integration-network + + # ============================================================================= + # WEBHOOK SERVER APPLICATION + # ============================================================================= + # Handles incoming webhooks from Unthread and processes them webhook-server: image: wgtechlabs/unthread-webhook-server:latest container_name: docker-unthread-webhook-server @@ -22,9 +53,9 @@ services: env_file: - .env environment: - - REDIS_URL=redis://redis:6379 + - REDIS_URL=redis://redis-webhook:6379 depends_on: - redis: + redis-webhook: condition: service_healthy restart: unless-stopped healthcheck: @@ -33,11 +64,21 @@ services: timeout: 10s retries: 3 start_period: 40s + networks: + - unthread-integration-network +# ============================================================================= +# PERSISTENT VOLUMES +# ============================================================================= +# Named volumes for data persistence across container restarts volumes: - redis_data: - driver: local + redis_webhook_data: # Redis webhook queue data +# ============================================================================= +# NETWORKING +# ============================================================================= +# External network for communication between services +# Create with: docker network create unthread-integration-network networks: - default: - name: unthread-integration-network + unthread-integration-network: + external: true diff --git a/package.json b/package.json index f65d65c..0595f73 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "unthread-webhook-server", - "version": "1.0.0-beta.4", - "description": "A Node.js server application that receives webhook events from Unthread.io and queues them for processing.", + "version": "1.0.0-beta.5", + "description": "A reliable, production-ready Node.js server for processing Unthread.io webhooks with signature verification and smart platform handling.", "license": "GPL-3.0", "private": true, "author": "WG Tech Labs (https://wgtechlabs.com)",