[WIP] Remote Build Plugin

builder/Dockerfile

@@ -0,0 +1,56 @@
+FROM python:3.5.7-slim-buster
+ENV PYTHONUNBUFFERED 1
+ENV DEBIAN_FRONTEND noninteractive
+ENV MESSAGELEVEL QUIET
+
+ARG ENABLE_REMOTEBUILD=true
+ARG ENABLE_PAM=true
+
+################################################################################
+# CORE
+# Do not modify this section
+
+RUN apt-get update && apt-get install -y \
+    net-tools \
+    build-essential \
+    libssl-dev \
+    uuid-dev \
+    libgpgme11-dev \
+    squashfs-tools \
+    libseccomp-dev \
+    wget \
+    pkg-config \
+    git \
+    cryptsetup \
+    debootstrap \
+    yum
+
+# Install websocket requisite for remote build
+RUN export VERSION=1.13.4 OS=linux ARCH=amd64 && \
+  wget https://dl.google.com/go/go$VERSION.$OS-$ARCH.tar.gz && \
+  tar -C /usr/local -xzvf go$VERSION.$OS-$ARCH.tar.gz && \
+  rm go$VERSION.$OS-$ARCH.tar.gz; \
+export VERSION=3.5.0 && \
+    wget https://github.com/sylabs/singularity/releases/download/v${VERSION}/singularity-${VERSION}.tar.gz && \
+    tar -xzf singularity-${VERSION}.tar.gz -C /tmp && \
+        rm singularity-${VERSION}.tar.gz; \
+cd /tmp/singularity && export  PATH=/usr/local/go/bin:$PATH && ./mconfig && \
+    make -C builddir && \
+    make -C builddir install
+
+# Install Python requirements out of /tmp so not triggered if other contents of /code change
+ADD builder/requirements.txt /tmp/requirements.txt
+RUN pip install --upgrade pip
+RUN pip install -r /tmp/requirements.txt
+# Install PAM Authentication (uncomment if wanted)
+RUN if $ENABLE_PAM; then pip install django-pam ; fi;
+
+ADD . /code/
+
+WORKDIR /code
+
+RUN apt-get autoremove -y
+RUN apt-get clean
+RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+EXPOSE 3032

builder/docker-compose.yml

@@ -0,0 +1,72 @@
+db:
+  image: postgres
+
+uwsgi:
+  restart: always
+  image: quay.io/vanessa/sregistry
+  volumes:
+    - .:/code
+    - ./static:/var/www/static
+    - ./images:/var/www/images
+    # uncomment for PAM auth
+    #- /etc/passwd:/etc/passwd 
+    #- /etc/shadow:/etc/shadow
+  links:
+    - redis
+    - db
+
+nginx:
+  restart: always
+  image: quay.io/vanessa/sregistry_nginx
+  ports:
+    - "80:80"
+    - "443:443"
+  volumes:
+    - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro
+    - ./uwsgi_params.par:/etc/nginx/uwsgi_params.par:ro
+    - /etc/ssl/certs:/etc/ssl/certs:ro
+    - /etc/ssl/private:/etc/ssl/private:ro
+  volumes_from:
+    - uwsgi
+  links:
+    - builder
+    - uwsgi
+    - db
+
+redis:
+  restart: always
+  image: redis:latest
+
+scheduler:
+  image: quay.io/vanessa/sregistry
+  command: python /code/manage.py rqscheduler
+  volumes:
+    - .:/code
+  volumes_from:
+    - uwsgi
+  links:
+    - redis
+    - db
+
+worker:
+  image: quay.io/vanessa/sregistry
+  command: python /code/manage.py rqworker default
+  volumes:
+    - .:/code
+  volumes_from:
+    - uwsgi
+  links:
+    - redis
+    - db
+
+# uncomment for remote build
+builder:
+  image: quay.io/vanessa/sregistry_builder
+  command: daphne --root-path "/v1/build-ws" -b 0.0.0.0 -p 3032 --proxy-headers shub.plugins.remote_build.asgi:application
+  volumes:
+    - .:/code
+  volumes_from:
+    - uwsgi
+  links:
+    - redis
+    - db

builder/nginx.conf.ws

@@ -0,0 +1,81 @@
+upstream websocket {
+    ip_hash;
+    server builder:3032 fail_timeout=0;
+}
+
+server {
+  listen                *:80;
+  server_name           localhost;
+
+  client_max_body_size 10024M;
+  client_body_buffer_size 10024M;
+  client_body_timeout 120;
+
+  add_header X-Clacks-Overhead "GNU Terry Pratchett";
+  add_header X-Clacks-Overhead "GNU Terry Pratchet";
+  add_header Access-Control-Allow-Origin *;
+  add_header 'Access-Control-Allow-Credentials' 'true';
+  add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+  add_header 'Access-Control-Allow-Headers' 'Authorization,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+
+  location /images {
+    alias /var/www/images;
+  }
+
+  location ~* \.(php|aspx|myadmin|asp)$ {
+    deny all;
+  }
+
+  location / {
+    include /etc/nginx/uwsgi_params.par;
+    uwsgi_pass uwsgi:3031;
+    uwsgi_max_temp_file_size 10024m;
+  }
+
+  location /static {
+    alias /var/www/static;
+  }
+
+  # Upload form should be submitted to this location
+  location /upload {
+
+        # Pass altered request body to this location
+        upload_pass   /api/uploads/complete/;
+
+        # Store files to this directory
+        # The directory is hashed, subdirectories 0 1 2 3 4 5 6 7 8 9 should exist
+        upload_store /var/www/images/_upload 1;        
+        upload_store_access user:rw group:rw all:rw;
+
+        # Set specified fields in request body
+        upload_set_form_field $upload_field_name.name "$upload_file_name";
+        upload_set_form_field $upload_field_name.content_type "$upload_content_type";
+        upload_set_form_field $upload_field_name.path "$upload_tmp_path";
+
+        # Inform backend about hash and size of a file
+        upload_aggregate_form_field "$upload_field_name.md5" "$upload_file_md5";
+        upload_aggregate_form_field "$upload_field_name.size" "$upload_file_size";
+
+        upload_pass_form_field "^submit$|^description$";
+        upload_pass_form_field "^SREGISTRY_EVENT$";
+        upload_pass_form_field "^collection$";
+        upload_pass_form_field "^name$";
+        upload_pass_form_field "^tag$";
+        upload_cleanup 400-599;
+
+    }
+
+    location /v1/build-ws/ {
+        proxy_pass http://websocket; # daphne (ASGI) listening on port 3032
+        proxy_http_version 1.1;
+        proxy_read_timeout 86400;
+        proxy_redirect     off;
+
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Host $server_name;
+   }
+}

builder/nginx.conf.wss

@@ -0,0 +1,163 @@
+upstream websocket {
+    ip_hash;
+    server builder:3032 fail_timeout=0;
+}
+
+server {
+  listen                *:80;
+  server_name           localhost;
+
+  client_max_body_size 10024M;
+  client_body_buffer_size 10024M;
+  client_body_timeout 120;
+
+  add_header X-Clacks-Overhead "GNU Terry Pratchett";
+  add_header X-Clacks-Overhead "GNU Terry Pratchet";
+  add_header Access-Control-Allow-Origin *;
+  add_header 'Access-Control-Allow-Credentials' 'true';
+  add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+  add_header 'Access-Control-Allow-Headers' 'Authorization,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+
+  location /images {
+    alias /var/www/images;
+  }
+
+  location ~* \.(php|aspx|myadmin|asp)$ {
+    deny all;
+  }
+
+  location / {
+    include /etc/nginx/uwsgi_params.par;
+    uwsgi_pass uwsgi:3031;
+    uwsgi_max_temp_file_size 10024m;
+  }
+
+  location /static {
+    alias /var/www/static;
+  }
+
+  # Upload form should be submitted to this location
+  location /upload {
+
+        # Pass altered request body to this location
+        upload_pass   /api/uploads/complete/;
+
+        # Store files to this directory
+        # The directory is hashed, subdirectories 0 1 2 3 4 5 6 7 8 9 should exist
+        upload_store /var/www/images/_upload 1;        
+        upload_store_access user:rw group:rw all:rw;
+
+        # Set specified fields in request body
+        upload_set_form_field $upload_field_name.name "$upload_file_name";
+        upload_set_form_field $upload_field_name.content_type "$upload_content_type";
+        upload_set_form_field $upload_field_name.path "$upload_tmp_path";
+
+        # Inform backend about hash and size of a file
+        upload_aggregate_form_field "$upload_field_name.md5" "$upload_file_md5";
+        upload_aggregate_form_field "$upload_field_name.size" "$upload_file_size";
+
+        upload_pass_form_field "^submit$|^description$";
+        upload_pass_form_field "^SREGISTRY_EVENT$";
+        upload_pass_form_field "^collection$";
+        upload_pass_form_field "^name$";
+        upload_pass_form_field "^tag$";
+        upload_cleanup 400-599;
+
+    }
+
+    location /v1/build-ws/ {
+        proxy_pass http://websocket; # daphne (ASGI) listening on port 3032
+        proxy_http_version 1.1;
+        proxy_read_timeout 86400;
+        proxy_redirect     off;
+
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Host $server_name;
+   }
+}
+
+server {
+
+    listen 443;
+    server_name localhost;
+
+    root html;
+    client_max_body_size 10024M;
+    client_body_buffer_size 10024M;
+
+    ssl on;
+    ssl_certificate /etc/ssl/certs/chained.pem;
+    ssl_certificate_key /etc/ssl/private/domain.key;
+    ssl_session_timeout 5m;
+    ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+    ssl_ciphers ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA;
+    ssl_session_cache shared:SSL:50m;
+    ssl_dhparam /etc/ssl/certs/dhparam.pem;
+    ssl_prefer_server_ciphers on;
+
+    location /images {
+        alias /var/www/images;
+    }
+
+    location /static {
+        alias /var/www/static;
+    }
+
+    location ~* \.(php|aspx|myadmin|asp)$ {
+      deny all;
+    }
+
+    # Upload form should be submitted to this location
+    location /upload {
+
+        # Pass altered request body to this location
+        upload_pass   /api/uploads/complete/;
+
+        # Store files to this directory
+        # The directory is hashed, subdirectories 0 1 2 3 4 5 6 7 8 9 should exist
+        upload_store /var/www/images/_upload 1;        
+        upload_store_access user:rw group:rw all:rw;
+
+        # Set specified fields in request body
+        upload_set_form_field $upload_field_name.name "$upload_file_name";
+        upload_set_form_field $upload_field_name.content_type "$upload_content_type";
+        upload_set_form_field $upload_field_name.path "$upload_tmp_path";
+
+        # Inform backend about hash and size of a file
+        upload_aggregate_form_field "$upload_field_name.md5" "$upload_file_md5";
+        upload_aggregate_form_field "$upload_field_name.size" "$upload_file_size";
+
+        upload_pass_form_field "^submit$|^description$";
+        upload_pass_form_field "^SREGISTRY_EVENT$";
+        upload_pass_form_field "^collection$";
+        upload_pass_form_field "^name$";
+        upload_pass_form_field "^tag$";
+        upload_cleanup 400-599;
+
+    }
+
+    location / {
+            include /etc/nginx/uwsgi_params.par;
+            uwsgi_pass uwsgi:3031;
+            uwsgi_max_temp_file_size 10024m;
+        }
+
+    location /v1/build-ws/ {
+        proxy_pass http://websocket; # daphne (ASGI) listening on port 3032
+        proxy_http_version 1.1;
+        proxy_read_timeout 86400;
+        proxy_redirect     off;
+
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Host $server_name;
+   }
+}
+

builder/requirements.txt

@@ -0,0 +1,15 @@
+django==2.2.8
+djangorestframework==3.10.3
+channels
+channels_redis
+django_user_agents
+social-auth-app-django
+django-crispy-forms
+django-rq
+rq-scheduler
+django-gravatar2
+django-extensions
+django-rest-swagger
+django-taggit
+django-taggit-templatetags
+psycopg2-binary