Promote Docker-based deployments; Replace SIdekiq

.github/workflows/continuous_integration.yml

@@ -8,10 +8,11 @@ on:
 
 env:
   BROWSERSLIST_IGNORE_OLD_DATA: beQuiet
-  DATABASE_URL: "postgres://postgres:postgres@localhost:5432/manifold_test"
   DISABLE_SPRING: yes
   PATH: /usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin:/usr/local/sbin
   RAILS_ENV: test
+  DATABASE_URL: "postgres://postgres:postgres@localhost:5432/manifold_test"
+  CACHE_DATABASE_URL: "postgres://postgres:postgres@localhost:5432/manifold_cache_test"
 
 jobs:
   client-lint:
@@ -59,14 +60,8 @@ jobs:
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: manifold_test
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
 
-      redis:
-        image: redis:6.2.7-alpine
-        ports: ["6379:6379"]
-        options: --entrypoint redis-server
-
     steps:
       - name: "Checkout code"
         uses: actions/checkout@v3
@@ -82,7 +77,7 @@ jobs:
 
       - name: "Setup test database"
         working-directory: "./api"
-        run: bin/rails db:setup
+        run: bin/rails db:test:prepare
 
       - name: "Run API specs"
         working-directory: "./api"

.github/workflows/docker.yml

@@ -0,0 +1,113 @@
+name: "Build & Push Docker"
+
+on:
+  push:
+    branches:
+      - next # Next, tags as "next"
+      - master # Edge, tags as "edge"
+      - release # Production, tags as "latest"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_name }}
+  cancel-in-progress: true
+
+env:
+  GHCR_REGISTRY: ghcr.io
+  GHCR_IMAGE_NAME: ${{ github.repository }}
+  MANAGED_REGISTRY: registry.digitalocean.com
+  MANAGED_IMAGE_NAME: manifold/manifold
+
+jobs:
+  build-api:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Login GHCR
+        uses: docker/login-action@v4
+        with:
+          registry: ${{ env.GHCR_REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Login DOCR
+        uses: docker/login-action@v4
+        with:
+          registry: ${{ env.MANAGED_REGISTRY }}
+          username: docr
+          password: ${{ secrets.DOCR_TOKEN }}
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE_NAME }}-api
+            ${{ env.MANAGED_REGISTRY }}/${{ env.MANAGED_IMAGE_NAME }}-api
+          tags: |
+            type=raw,value=latest,enable=${{ github.ref_name == 'release' }}
+            type=raw,value=edge,enable={{ is_default_branch }}
+            type=ref,event=branch
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push Docker
+        id: push
+        uses: docker/build-push-action@v5
+        with:
+          context: ./api
+          target: production
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args:
+            "RAILS_ENV=production"
+
+  build-client:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Login GHCR
+        uses: docker/login-action@v4
+        with:
+          registry: ${{ env.GHCR_REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Login DOCR
+        uses: docker/login-action@v4
+        with:
+          registry: ${{ env.MANAGED_REGISTRY }}
+          username: docr
+          password: ${{ secrets.DOCR_TOKEN }}
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE_NAME }}-client
+            ${{ env.MANAGED_REGISTRY }}/${{ env.MANAGED_IMAGE_NAME }}-client
+          tags: |
+            type=raw,value=latest,enable=${{ github.ref_name == 'release' }}
+            type=raw,value=edge,enable={{ is_default_branch }}
+            type=ref,event=branch
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push Docker
+        id: push
+        uses: docker/build-push-action@v5
+        with:
+          context: ./client
+          target: production
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args:
+            "RAILS_ENV=production"

.github/workflows/release.yml

@@ -0,0 +1,22 @@
+name: Production Release
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Update Release Branch
+    runs-on: ubuntu-latest
+    steps:
+      - name: "checkout release-candidate"
+        uses: actions/checkout@v4
+        with:
+          ref: release
+
+      - name: "Push release changes"
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          git fetch
+          git reset --hard origin/master
+          git push --force origin release

.gitignore

@@ -64,3 +64,5 @@ backup-local*.tar
 /docker/minio/client/*
 !/docker/minio/client/.keep
 !/docker/minio/client/initialize.sh
+
+CLAUDE.md

api/.gitignore

@@ -40,3 +40,5 @@ backup*
 
 /config/*.local.yml
 /config/credentials/local.*
+
+.env

api/Dockerfile

@@ -0,0 +1,69 @@
+FROM ruby:3.2-bullseye AS base
+
+RUN mkdir -pv /bundle/bin
+
+ENV BUNDLE_PATH=/bundle \
+    BUNDLE_BIN=/bundle/bin \
+    GEM_HOME=/bundle \
+    RACK_ENV=development \
+    RAILS_ENV=development \
+    RAILS_LOG_TO_STDOUT=true \
+    RAILS_SERVE_STATIC_FILES=true \
+    RAILS_MIN_THREADS=16 \
+    RAILS_MAX_THREADS=16 \
+    WORKER_COUNT=0
+
+COPY docker/install_node_16.sh /usr/local/src/install_node_16.sh
+
+RUN bash /usr/local/src/install_node_16.sh
+
+RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    curl \
+    ghostscript \
+    gnupg gnupg2 \
+    imagemagick \
+    libglib2.0-0 libglib2.0-dev \
+    libicu-dev \
+    libjemalloc2 \
+    libpoppler-glib8 \
+    librsvg2-bin \
+    libsndfile1-dev \
+    libvips \
+    libvips-dev \
+    mediainfo \
+    nodejs \
+    postgresql-common \
+    pandoc
+
+RUN DEBIAN_FRONTEND=noninteractive /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y
+
+RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends postgresql-client-13
+
+RUN npm install -g mammoth@^1.4.16 wscat@^6.0.1
+
+RUN sed -i '/<policy domain="coder" rights="none" pattern="PDF" \/>/d' /etc/ImageMagick-6/policy.xml
+
+COPY ./ /srv/app/
+
+WORKDIR /srv/app
+COPY Gemfile /srv/app/Gemfile
+COPY Gemfile.lock /srv/app/Gemfile.lock
+
+COPY docker/entrypoint.sh /usr/bin/
+RUN chmod +x /usr/bin/entrypoint.sh
+ENTRYPOINT ["entrypoint.sh"]
+
+ENV MAMMOTH_PATH=/usr/bin/mammoth
+ENV PATH="${BUNDLE_BIN}:${PATH}"
+ENV LD_PRELOAD=libjemalloc.so.2
+
+RUN bundle install
+
+CMD ["bin/puma", "-C", "config/puma.rb"]
+
+# There are currently no differences between dev and prod Dockerfiles, but these are here to provide parity with the client Dockerfile
+FROM base AS development
+
+FROM base AS production

api/Gemfile

@@ -6,8 +6,6 @@ gem "aasm", "~> 4.0"
 gem "absolute_time", "~> 1.0.0"
 gem "active_interaction", "~> 4.0"
 gem "activejob-retry", "~> 0.4"
-gem "activejob-traffic_control", "~>0.1.0"
-gem "activejob-uniqueness", "~> 0.2.4"
 gem "activerecord-import", "~> 1.4.1"
 gem "active_record_upsert", "~> 0.11.1"
 gem "acts_as_list", "~> 1.2.4"
@@ -29,6 +27,7 @@ gem "cleanroom"
 gem "closure_tree", "~> 7.0"
 # Until Rails 7.1
 gem "composite_primary_keys", "~> 14.0.10"
+gem "connection_pool"
 gem "crass", "~> 1.0.5"
 gem "csl-styles", "~> 1.0"
 gem "cssbeautify"
@@ -55,6 +54,7 @@ gem "fast_jsonapi", git: "https://github.com/ManifoldScholar/fast_jsonapi.git",
 gem "filesize", "~> 0.1"
 gem "format_engine", "~> 0.7.7"
 gem "friendly_id", "~> 5.0"
+gem "solid_cache", "~> 0.7"
 gem "fuzzy_match", "~> 2.1.0"
 gem "geocoder", "1.8.2"
 gem "gepub", "~> 1.0.4"
@@ -64,7 +64,7 @@ gem "hashie", "~> 3.0"
 gem "has_scope", "~> 0.8.1"
 gem "httparty", "~> 0.21.0"
 gem "image_processing", "~> 1.12"
-gem "job-iteration", "~> 1.10.0"
+gem "job-iteration", "~> 1.11.0"
 gem "jsonb_accessor", "~> 1.0"
 gem "jwt", "~> 1.5"
 gem "kaminari", "~> 1.2"
@@ -80,7 +80,7 @@ gem "namae", "~>1.0"
 gem "naught", "~> 1.0"
 gem "net-sftp", "~> 2.1.2"
 gem "net-ssh", "~> 5.2.0"
-gem "nokogiri", "~> 1.15.4"
+gem "nokogiri", "~> 1.19.1"
 gem "oauth2", "~> 1.3"
 gem "oj", "~> 3.16"
 gem "omniauth", "~> 2.1.3"
@@ -104,17 +104,13 @@ gem "rack-cors", "~> 1.0"
 gem "rails", "~> 7.0"
 gem "rainbow", "~> 3.0"
 gem "redcarpet", "~> 3.5"
-gem "redis", "~> 4.5"
-gem "redis-namespace", "~> 1.0"
-gem "redis-objects", "~> 1.4"
 gem "rolify", "~> 5.1"
 gem "rubyzip", "~> 2.3.1"
 gem "scanf", "~> 1.0.0"
 gem "scenic", "~> 1.4"
 gem "shrine", "~> 3.5.0"
 gem "shrine-google_cloud_storage", "~> 3.3"
 gem "shrine-tus", "~> 2.0"
-gem "sidekiq", "< 6"
 gem "signet", "~> 0.10"
 gem "sinatra", "~>2.2"
 gem "sixarm_ruby_unaccent", "~> 1.2.2"
@@ -134,7 +130,6 @@ gem "validates_email_format_of", "~> 1.0"
 gem "validate_url", "~> 1.0"
 gem "with_advisory_lock", "~> 4.0"
 gem "zaru", "~> 1.0.0"
-gem "zhong", "~> 0.3"
 
 group :development, :test do
   gem "pry-byebug"
@@ -170,7 +165,8 @@ end
 
 group :test do
   gem "database_cleaner-active_record", "~> 2.1.0"
-  gem "database_cleaner-redis", "~> 2.0"
   gem "test-prof", "~> 1.0"
   gem "with_model", "~> 2.1"
 end
+
+gem "good_job", "~> 3.99"