Docker build and other cleanups

.circleci/config.yml

@@ -15,64 +15,24 @@ jobs:
     resource_class: large
     steps:
       - run:
-          name: docker-compose version
-          command: docker-compose --version
+          name: docker compose version
+          command: docker compose version
+      - checkout
       - run:
+          name: create coverage directory
           command: |
-            git clone https://github.com/metacpan/metacpan-docker.git
-            cd metacpan-docker
-          name: metacpan-docker checkout
-      - checkout:
-          path: metacpan-docker/src/metacpan-api
+            mkdir cover_db
+            chmod o+w cover_db
       - run:
+          name: docker compose build
           command: |
-            pushd metacpan-docker
-            ./bin/metacpan-docker init
-          name: clone missing repositories
+            docker compose --profile test build api-test
       - run:
-          command: |
-            pushd metacpan-docker
-            docker-compose build --build-arg CPM_ARGS='--with-test' api_test
-          name: compose build
-      - run:
-          command: |
-            pushd metacpan-docker
-            ./bin/metacpan-docker init
-            docker-compose --verbose up -d api_test
-          name: compose up
-      - run:
-          command: |
-            pushd metacpan-docker
-            docker-compose exec -T api_test cpm install -g Devel::Cover
-          name: install Devel::Cover
-      # Since we're running docker-compose -d, we don't actually know if
-      # Elasticsearch is available at the time this build step begins. We
-      # probably need to wait for it here, so we'll add our own check.
-      - run:
-          command: |
-            pushd metacpan-docker
-            ./src/metacpan-api/wait-for-es.sh http://localhost:9200 elasticsearch_test
-          name: wait for ES
-      - run:
-          command: |
-            pushd metacpan-docker
-            docker-compose exec -T api_test env HARNESS_PERL_SWITCHES="-MDevel::Cover=+ignore,^t/|^test-data/|^etc/" prove -lr --jobs 4 t
           name: run tests with coverage
+          command: |
+            docker compose --profile test run --env HARNESS_PERL_SWITCHES=-MDevel::Cover -v ./cover_db:/app/cover_db/ api-test bash -c 'prove -lr -j4 t && cover -report json'
       # We are relying on environment variables from the host to be available when
       # we publish the report, so we publish from the host rather than trying
       # to propagate env variables to the container.
-      - run:
-          command: |
-            pushd metacpan-docker
-            docker-compose exec -T api_test cover -report json
-          name: create coverage report
       - codecov/upload:
-          file: metacpan-docker/src/metacpan-api/cover_db/cover.json
-      - run:
-          command: |
-            pushd metacpan-docker
-            docker-compose logs
-            docker stats --no-stream
-            docker ps -a | head
-          name: docker-compose logs
-          when: on_fail
+          file: cover_db/cover.json

.github/workflows/build-container.yml

@@ -0,0 +1,46 @@
+name: Build container
+on:
+  push:
+    branches:
+      - master
+      - staging
+      - prod
+  pull_request:
+    types: [opened, synchronize, labeled]
+    branches:
+      - master
+  workflow_dispatch:
+jobs:
+  docker-build:
+    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'build-container')
+    runs-on: ubuntu-22.04
+    name: Docker Build and Push
+    steps:
+      - name: Generate Auth Token
+        uses: actions/create-github-app-token@v2
+        id: app-token
+        with:
+          app-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          owner: metacpan
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ steps.app-token.outputs.token }}
+      - uses: metacpan/metacpan-actions/docker-build-push@master
+        id: build-push
+        with:
+          docker_hub_username: ${{ secrets.DOCKER_HUB_USER }}
+          docker_hub_password: ${{ secrets.DOCKER_HUB_TOKEN }}
+          ghcr_username: ${{ github.repository_owner }}
+          ghcr_password: ${{ secrets.GITHUB_TOKEN }}
+          test-target: test
+#      - name: Update deployed image
+#        if: steps.find-tag-names.outputs.latest
+#        uses: metacpan/metacpan-actions/update-deployed-tag:master
+#        with:
+#          token: ${{ steps.app-token.outputs.token }}
+#          app: api
+#          environment: prod
+#          base-tag: ${{ steps.find-tag-names.outputs.latest }}
+#          tag: ${{ steps.find-tag-names.outputs.sha }}
+

.github/workflows/code-formatting.yml

@@ -0,0 +1,52 @@
+---
+name: Code Formatting
+on:
+  push:
+    branches:
+      - 'master'
+  merge_group:
+  pull_request:
+    branches:
+      - '*'
+  workflow_dispatch:
+
+jobs:
+  code-formatting:
+    runs-on: ubuntu-24.04
+    name: Code Formatting
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Fetch base ref
+        if: ${{ github.event.pull_request }}
+        run: git fetch origin ${{ github.base_ref }}:upstream
+      - name: Install Carton
+        uses: perl-actions/install-with-cpm@v1
+        with:
+          install: Carton
+      - name: Install CPAN deps
+        uses: perl-actions/install-with-cpm@v1
+        with:
+          cpanfile: 'cpanfile'
+          args: >
+            --resolver=snapshot
+            --without-runtime
+            --without-test
+            --without-build
+            --with-develop
+      - name: Install precious
+        run: ./bin/install-precious /usr/local/bin
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+      - run: perltidy --version
+      - name: Select files
+        id: select-files
+        run: |
+          if [[ -n "${{ github.event.pull_request.number }}" ]]; then
+            echo 'precious-args=--git-diff-from upstream' >> "$GITHUB_OUTPUT"
+          else
+            echo 'precious-args=--all' >> "$GITHUB_OUTPUT"
+          fi
+      - name: Lint files
+        run: precious lint ${{ steps.select-files.outputs.precious-args }}

.gitignore

@@ -10,6 +10,9 @@
 /perltidy.LOG
 /pm_to_blib
 /var
+/bin/omegasort
+/bin/precious
+/bin/ubi
 /etc/metacpan_local.pl
 /t/var/darkpan/
 /t/var/log/

Dockerfile

@@ -1,24 +1,101 @@
-# hadolint ignore=DL3007
-FROM metacpan/metacpan-base:latest
-
-COPY cpanfile cpanfile.snapshot /metacpan-api/
-WORKDIR /metacpan-api
-
-# CPM installations of dependencies does not install or run tests. This is
-# because the modules themselves have been tested, and the metacpan use of the
-# modules is tested by the test suite. Removing the tests, reduces the overall
-# size of the images.
-RUN mkdir /CPAN \
-    && apt-get update \
-    && apt-get satisfy -y --no-install-recommends 'rsync (>= 3.2.3)' 'jq (>= 1.6)' \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* \
-    && cpm install --global \
-    && git config --global --add safe.directory /metacpan-api \
-    && rm -fr /root/.cpanm /root/.perl-cpm /var/cache/apt/lists/* /tmp/*
+ARG SLIM_BUILD
+ARG MAYBE_BASE_BUILD=${SLIM_BUILD:+server-base-slim}
+ARG BASE_BUILD=${MAYBE_BASE_BUILD:-server-base}
+
+################### Web Server Base
+FROM metacpan/metacpan-base:main-20250531-090128 AS server-base
+FROM metacpan/metacpan-base:main-20250531-090129-slim AS server-base-slim
+
+################### CPAN Prereqs
+FROM server-base AS build-cpan-prereqs
+SHELL [ "/bin/bash", "-euo", "pipefail", "-c" ]
+
+WORKDIR /app/
+
+COPY cpanfile cpanfile.snapshot ./
+RUN \
+    --mount=type=cache,target=/root/.perl-cpm,sharing=private \
+<<EOT
+    cpm install --show-build-log-on-failure --resolver=snapshot
+EOT
+
+################### Web Server
+# false positive
+# hadolint ignore=DL3006
+FROM ${BASE_BUILD} AS server
+SHELL [ "/bin/bash", "-euo", "pipefail", "-c" ]
+
+WORKDIR /app/
+
+COPY app.psgi log4perl* metacpan_server.* metacpan_server_local.* ./
+COPY es es
+COPY bin bin
+COPY lib lib
+COPY root root
+
+COPY --from=build-cpan-prereqs /app/local local
+
+ENV PERL5LIB="/app/local/lib/perl5"
+ENV PATH="/app/local/bin:${PATH}"
+ENV METACPAN_SERVER_HOME=/app
 
 VOLUME /CPAN
 
-EXPOSE 5000
+USER metacpan
+
+CMD [ \
+    "/uwsgi.sh", \
+    "--http-socket", ":8000" \
+]
+
+EXPOSE 8000
+
+HEALTHCHECK --start-period=3s CMD [ "curl", "--fail", "http://localhost:8000/healthcheck" ]
+
+################### Development Server
+FROM server AS develop
+SHELL [ "/bin/bash", "-euo", "pipefail", "-c" ]
+
+ENV COLUMNS=120
+ENV PLACK_ENV=development
+
+USER root
+
+COPY cpanfile cpanfile.snapshot ./
+
+RUN \
+    --mount=type=cache,target=/root/.perl-cpm \
+<<EOT
+    cpm install --show-build-log-on-failure --resolver=snapshot --with-develop
+    chown -R metacpan:users ./
+EOT
+
+USER metacpan
+
+################### Test Runner
+FROM develop AS test
+SHELL [ "/bin/bash", "-euo", "pipefail", "-c" ]
+
+ENV PLACK_ENV=
+
+USER root
+
+RUN \
+    --mount=type=cache,target=/root/.perl-cpm \
+<<EOT
+    cpm install --show-build-log-on-failure --resolver=snapshot --with-test
+EOT
+
+COPY .perlcriticrc .perltidyrc perlimports.toml precious.toml .editorconfig metacpan_server_testing.* ./
+COPY t t
+COPY test-data test-data
+
+RUN chown metacpan t/var
+
+USER metacpan
+CMD [ "prove", "-l", "-r", "-j", "2", "t" ]
+
+################### Production Server
+FROM server AS production
 
-CMD [ "/wait-for-it.sh", "${PGDB}",  "--", "${API_SERVER}", "./bin/api.pl" ]
+USER metacpan

bin/install-precious

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# This is for installing precious and other 3rd party libs needed for linting
+# in CI
+
+set -euo pipefail
+
+if [ -z "${1:-}" ]; then
+    echo "usage: ./bin/install-precious /path/to/bin/dir"
+    exit 1
+fi
+
+TARGET=$1
+export TARGET
+
+TARGET=$1
+export TARGET
+
+curl --silent --location \
+    https://raw.githubusercontent.com/houseabsolute/ubi/master/bootstrap/bootstrap-ubi.sh |
+    sh
+
+ubi --project houseabsolute/omegasort --in "$TARGET"
+ubi --project houseabsolute/precious --in "$TARGET"