feat: Adding Dockerfile for stats and CI for building container images

.github/workflows/ci.yml

@@ -0,0 +1,103 @@
+name: CI Pipeline
+
+on:
+  push:
+  pull_request:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  test-and-build-stats:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Log in to Container Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ${{ env.REGISTRY }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Extract metadata for stats image
+      id: meta-stats
+      uses: docker/metadata-action@v5
+      with:
+        images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/stats
+        tags: |
+          type=ref,event=branch
+          type=ref,event=pr
+          type=sha
+          type=raw,value=latest,enable={{is_default_branch}}
+
+    - name: Build stats Docker image
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        file: ./stats.Dockerfile
+        push: false
+        tags: |
+          ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/stats:${{ github.sha }}
+          ${{ steps.meta-stats.outputs.tags }}
+        labels: ${{ steps.meta-stats.outputs.labels }}
+
+    - name: Run unit tests
+      run: |
+        docker run --rm \
+          -v ${{ github.workspace }}/tests:/app/tests \
+          ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/stats:${{ github.sha }} \
+          python -m pytest -s tests/
+
+    - name: Push stats Docker image
+      if: success() && github.event.pull_request.head.repo.full_name == github.repository
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        file: ./stats.Dockerfile
+        push: true
+        tags: ${{ steps.meta-stats.outputs.tags }}
+        labels: ${{ steps.meta-stats.outputs.labels }}
+
+  build-site:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Log in to Container Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ${{ env.REGISTRY }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Extract metadata for site image
+      id: meta-site
+      uses: docker/metadata-action@v5
+      with:
+        images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/site
+        tags: |
+          type=ref,event=branch
+          type=ref,event=pr
+          type=sha
+          type=raw,value=latest,enable={{is_default_branch}}
+
+    - name: Build and push site Docker image
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        file: ./site.Dockerfile
+        push: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
+        tags: ${{ steps.meta-site.outputs.tags }}
+        labels: ${{ steps.meta-site.outputs.labels }}

README.md

@@ -129,6 +129,26 @@ To preview local changes, it's possible to serve the site locally:
 
    ... and then the site will be served on http://0.0.0.0:4000 instead.  (You will of course need to rebuild the Docker image after updating the Dockerfile.)
 
+
+Run via Container
+-----------------
+
+The whole workflow can be run as a container (docker or podman) including downloading stats files from Common Crawl's S3 bucket and generating new plots.
+
+```bash
+# clone the repository (to have the latest crawl IDs)
+git clone https://github.com/commoncrawl/cc-crawl-statistics.git
+cd cc-crawl-statistics
+
+# download stats and generate plots
+# SSH, AWS keys, and stats and plots directories must be mounted into the container
+podman run --rm -v ~/.ssh:/root/.ssh:ro -v ~/.aws:/root/.aws:ro -v $(pwd -P)/stats:/app/stats -v $(pwd -P)/plots:/app/plots ghcr.io/commoncrawl/cc-crawl-statistics/stats:latest 
+
+# if needed you can manually build the container image
+podman build -f stats.Dockerfile -t ghcr.io/commoncrawl/cc-crawl-statistics/stats:latest
+```
+
+
 Related Projects
 ----------------
 

get_stats_and_plot.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+echo "Starting ..."
+
+./get_stats.sh
+
+# make sure plot directories exist
+mkdir -p plots/crawler
+mkdir -p plots/crawloverlap
+mkdir -p plots/crawlsize
+mkdir -p plots/throughput
+mkdir -p plots/tld
+
+./plot.sh
+
+echo "Done."

requirements.txt

@@ -1,5 +1,10 @@
 hyperloglog==0.0.14
-isoweek
-mrjob
-tldextract
-ujson
+isoweek==1.3.3
+mrjob==0.7.4
+tldextract==5.1.2
+ujson==5.10.0
+
+# tests
+pytest
+jsonpickle
+setuptools

requirements_plot.txt

@@ -1,5 +1,6 @@
-ggplot
-idna
-pandas
-pygraphviz
-rpy2
+ggplot==0.11.5
+idna==3.7
+#pandas==2.1.4+dfsg
+pandas==2.1.4
+pygraphviz==1.13
+rpy2==3.5.15

stats.Dockerfile

@@ -0,0 +1,40 @@
+# Replicating pjox/cc-crawl-statistics
+FROM python:3.12
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    git \
+    python3-rpy2 \
+    r-cran-ggplot2 \
+    graphviz-dev \
+    r-base jq \
+    awscli
+
+# Set working directory
+WORKDIR /app
+
+# Copy dependency config files (first for cache)
+COPY requirements.txt .
+COPY requirements_plot.txt .
+
+# Install Python dependencies
+RUN pip3 install -r requirements.txt
+RUN pip3 install -r requirements_plot.txt
+
+# Copy the remaining repository files
+COPY stats/crawler ./stats/crawler
+COPY plots/ ./plots/
+COPY plot/ ./plot/
+COPY tests/ ./tests/
+
+COPY *.sh ./
+COPY *.py ./
+COPY _config.yml ./
+
+# Set PYTHONPATH environment variable
+ENV PYTHONPATH=/app
+
+# ggplot2 is already installed via r-cran-ggplot2 system package above
+
+# Default command
+CMD ["./get_stats_and_plot.sh"]