chore: release version v0.2.0

- Added Playwright service and configuration
- Added configurable Docker images
- Enhanced Redis persistence
- Updated environment templates

Author: amirasaran

.gitignore

@@ -20,6 +20,7 @@ ENV/
 !db.env.example
 !minio.env.example
 !frontend.env.example
+!playwright.env.example
 
 # Backup directories
 .config_backup/

CHANGELOG.md

@@ -0,0 +1,27 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+## [v0.2.0] - 2025-01-15
+
+### Added
+- Added Playwright service for web automation
+- Added Redis data volume for persistence
+- Added support for configurable Docker images through environment variables
+- Added playwright.env.example template
+
+### Changed
+- Updated Docker Compose configuration to use environment variables for image versions
+- Modified frontend Dockerfile to use configurable image from environment
+- Updated environment templates with new variables
+- Enhanced install script to handle missing environment variables
+
+### Infrastructure
+- Added configurable image versions for all services:
+  - WATER_CRAWL_BACKEND_IMAGE
+  - WATER_CRAWL_FRONTEND_IMAGE
+  - NEGINX_IMAGE
+  - MINIO_IMAGE
+  - POSTGRES_IMAGE
+  - REDIS_IMAGE
+  - PLAYWRIGHT_IMAGE

docker-compose.yml

@@ -2,6 +2,8 @@ x-app: &app
   build:
     context: watercrawl
     dockerfile: Dockerfile
+    args:
+      - WATER_CRAWL_BACKEND_IMAGE
   env_file:
     - app.env
   volumes:
@@ -26,12 +28,14 @@ services:
     build:
       context: frontend
       dockerfile: Dockerfile
+      args:
+        - WATER_CRAWL_FRONTEND_IMAGE
     restart: always
     env_file:
       - frontend.env
 
   nginx:
-    image: nginx:1.25-alpine
+    image: ${NEGINX_IMAGE:-nginx:1.25-alpine}
     restart: always
     ports:
       - "80:80"
@@ -43,7 +47,7 @@ services:
       - app
 
   # minio:
-  #   image: minio/minio:RELEASE.2024-11-07T00-52-20Z
+  #   image: {MINIO_IMAGE:-minio/minio:RELEASE.2024-11-07T00-52-20Z}
   #   restart: always
   #   volumes:
   #     - minio-data:/data
@@ -56,7 +60,7 @@ services:
   #     timeout: 20s
 
   postgres:
-    image: postgres:17.2-alpine3.21
+    image: ${POSTGRES_IMAGE:-postgres:17.2-alpine3.21}
     restart: always
     env_file:
       - db.env
@@ -67,9 +71,17 @@ services:
       interval: 5s
       timeout: 5s
 
+  playwright:
+    image: ${PLAYWRIGHT_IMAGE:-watercrawl/playwright:1.1}
+    restart: always
+    env_file:
+      - playwright.env
+
   redis:
-    image: redis:latest
+    image: ${REDIS_IMAGE:-redis:7.2-alpine}
     restart: always
+    volumes:
+      - redis-data:/data
 
 volumes:
   postgres-db:

env_templates/.env.example

@@ -1 +1,6 @@
-VERISON=latest
+WATER_CRAWL_BACKEND_IMAGE=watercrawl/watercrawl:latest
+WATER_CRAWL_FRONTEND_IMAGE=watercrawl/frontend:latest
+NEGINX_IMAGE=nginx:1.25-alpine
+MINIO_IMAGE=minio/minio:RELEASE.2024-11-07T00-52-20Z
+POSTGRES_IMAGE=postgres:17.2-alpine3.21
+REDIS_IMAGE=redis:7.2-alpine

env_templates/app.env.example

@@ -85,3 +85,5 @@ IS_GOOGLE_LOGIN_ACTIVE=False
 # EMAIL_HOST_PASSWORD=
 # DEFAULT_FROM_EMAIL=info@example.com
 
+PLAYWRIGHT_SERVER=http://playwright:8000
+PLAYWRIGHT_API_KEY=your-secret-api-key

env_templates/playwright.env.example

@@ -0,0 +1,3 @@
+AUTH_API_KEY=your_auth_api_key_here
+PORT=8000
+HOST=0.0.0.0

frontend/Dockerfile

@@ -1 +1,2 @@
-FROM watercrawl/frontend:latest
+ARG WATER_CRAWL_FRONTEND_IMAGE
+FROM ${WATER_CRAWL_FRONTEND_IMAGE}

install.sh

@@ -111,9 +111,24 @@ safe_replace() {
     fi
 }
 
+add_missing_variables() {
+    local example_file="$1"
+    local env_file="$2"
+
+    while IFS='=' read -r key value; do
+        # Skip lines that start with '#'
+        if [[ $key == \#* || -z $key ]]; then
+            continue
+        fi
+        if ! grep -q "^$key=" "$env_file"; then
+            echo "$key=$value" >> "$env_file"
+        fi
+    done < "$example_file"
+}
+
 # Function to validate required example files
 validate_example_files() {
-    local example_files=("env_templates/app.env.example" "env_templates/db.env.example" "env_templates/minio.env.example" "env_templates/frontend.env.example" "env_templates/.env.example")
+    local example_files=("env_templates/app.env.example" "env_templates/db.env.example" "env_templates/minio.env.example" "env_templates/frontend.env.example" "env_templates/playwright.env.example" "env_templates/.env.example")
 
     for file in "${example_files[@]}"; do
         if [[ ! -f "$file" ]]; then
@@ -125,7 +140,7 @@ validate_example_files() {
 
 # Function to validate environment files
 validate_env_files() {
-    local files=("app.env" "db.env" "frontend.env")
+    local files=("app.env" "db.env" "frontend.env" "playwright.env")
 
     for file in "${files[@]}"; do
         if [[ ! -f "$file" ]]; then
@@ -140,6 +155,7 @@ REINSTALL=false
 DEBUG=false
 DB_FILE_EXISTS=$([ -f db.env ] && echo "true" || echo "false")
 APP_FILE_EXISTS=$([ -f app.env ] && echo "true" || echo "false")
+PLAYWRIGHT_FILE_EXISTS=$([ -f playwright.env ] && echo "true" || echo "false")
 
 for arg in "$@"; do
     case $arg in
@@ -197,14 +213,22 @@ WEBSITE_URL="${WEBSITE_PROTOCOL}://${WEBSITE_DOMAIN}"
 # STORAGE_URL="${STORAGE_PROTOCOL}://${STORAGE_DOMAIN}"
 
 # Copy and configure environment files
-for env in app db frontend; do
+for env in app db frontend playwright; do
     if [[ -f "env_templates/${env}.env.example" && ! -f "${env}.env" ]]; then
         echo "Creating ${env}.env..."
         cp "env_templates/${env}.env.example" "${env}.env"
+    else
+        echo -e "${YELLOW}${env}.env already exists. checking for missing variables...${NC}"
+        add_missing_variables "env_templates/${env}.env.example" "${env}.env"
     fi
 
 done
 
+if [[ -f "env_templates/.env.example" && ! -f ".env" ]]; then
+    echo "Creating .env..."
+    cp "env_templates/.env.example" ".env"
+fi
+
 # Generate secrets and configure app.env
 if [[ -f app.env ]]; then
     safe_replace "app.env" "ALLOWED_HOSTS" "$WEBSITE_DOMAIN"
@@ -242,6 +266,16 @@ if [[ -f frontend.env ]]; then
     safe_replace "frontend.env" "VITE_API_URL" "$WEBSITE_URL"
 fi
 
+if [[ -f playwright.env ]]; then
+    if [[ "$PLAYWRIGHT_FILE_EXISTS" == "true" ]]; then
+        echo -e "\n${YELLOW}playwright.env already exists. Skipping API key generation.${NC}"
+    else
+        AUTH_API_KEY=$(generate_random_string 32)
+        safe_replace "playwright.env" "AUTH_API_KEY" "$AUTH_API_KEY"
+        safe_replace "app.env" "PLAYWRIGHT_API_KEY" "$AUTH_API_KEY"
+    fi
+fi
+
 # Validate environment files
 validate_env_files
 

watercrawl/Dockerfile

@@ -1,4 +1,5 @@
-FROM watercrawl/watercrawl:latest
+ARG WATER_CRAWL_BACKEND_IMAGE
+FROM ${WATER_CRAWL_BACKEND_IMAGE}
 
 COPY plugin_requirements.txt /var/www/plugin_requirements.txt