diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..aaf1a14 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,49 @@ +name: Deploy to static server + +on: + push: + branches: + - ep2024 + schedule: + - cron: "*/10 * * * *" # every 10 minutes + workflow_dispatch: + +jobs: + tests: + name: Run tests + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python 3 + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Setup uv + uses: astral-sh/setup-uv@v5 + + - name: Install dependencies from uv.lock + run: make deps/install + + - name: Download data + run: uv run make download > /dev/null 2>&1 + env: + PRETALX_TOKEN: ${{ secrets.PRETALX_TOKEN }} + + - name: Transform data + run: uv run make transform > /dev/null 2>&1 + + - name: Setup SSH + uses: webfactory/ssh-agent@v0.9.1 + with: + ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} + + - name: Adjust known_hosts + run: ssh-keyscan "static.europython.eu" > ~/.ssh/known_hosts + + - name: Deploy + run: uv run make deploy FORCE_DEPLOY=true diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index a9b7831..0000000 --- a/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM python:3.12 - -WORKDIR /srv - -COPY requirements.txt . -RUN pip install -r requirements.txt - -COPY src/ ./src/ -COPY Makefile . - -RUN mkdir -p /srv/data/raw/europython-2024/ -RUN mkdir -p /srv/data/public/europython-2024/ - - -CMD ["make", "all"] diff --git a/Makefile b/Makefile index d7ca424..09440dd 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,20 @@ +# Variables for the project +# ========================= +CONFERENCE ?= ep2024 +DATA_DIR ?= ./data/public/$(CONFERENCE)/ + +# Variables for remote host +# ========================= +VPS_USER ?= static_content_user +VPS_HOST ?= static.europython.eu +VPS_PATH ?= /home/$(VPS_USER)/content/programapi/$(CONFERENCE)/releases +REMOTE_CMD=ssh $(VPS_USER)@$(VPS_HOST) + +# Variables for deploy +# ========================== +TIMESTAMP ?= $(shell date +%Y%m%d%H%M%S) +FORCE_DEPLOY ?= false + dev: uv sync --dev @@ -21,6 +38,16 @@ endif all: download transform +ifeq ($(FORCE_DEPLOY), true) +deploy: TARGET = $(VPS_PATH)/$(TIMESTAMP) +deploy: + @echo "\n\n**** Deploying branch '$(CONFERENCE)' to $(TARGET)...\n\n" + $(REMOTE_CMD) "mkdir -p $(TARGET)" + rsync -avz --delete $(DATA_DIR) $(VPS_USER)@$(VPS_HOST):$(TARGET) + $(REMOTE_CMD) "cd $(VPS_PATH) && ln -snf $(TIMESTAMP) current" + @echo "\n\n**** Deployment complete.\n\n" +endif + test: uv run pytest diff --git a/deploy/.python-version b/deploy/.python-version deleted file mode 100644 index 2c07333..0000000 --- a/deploy/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.11 diff --git a/deploy/Makefile b/deploy/Makefile deleted file mode 100644 index 8b38e19..0000000 --- a/deploy/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -run-playbook: - ansible-playbook -i hosts.ini playbook.yml --extra-vars "app_version=$(V)" - -deploy/deps/init: - pip install pip-tools - -deploy/deps/compile: - pip-compile - -deploy/deps/install: - pip-sync diff --git a/deploy/docker-compose.yml.j2 b/deploy/docker-compose.yml.j2 deleted file mode 100644 index 1aadee7..0000000 --- a/deploy/docker-compose.yml.j2 +++ /dev/null @@ -1,28 +0,0 @@ -version: '3' - -services: - app: - build: . - env_file: - - env/config - volumes: - - ./data/static:/srv/data/public - - nginx: - image: nginx:latest - ports: - - "80:80" - - "443:443" - volumes: - - ./nginx.conf:/etc/nginx/nginx.conf - - ./data/certbot/conf:/etc/letsencrypt - - ./data/certbot/www:/var/www/certbot - - ./data/static:/usr/share/static - depends_on: - - app - - certbot: - image: certbot/certbot - volumes: - - ./data/certbot/conf:/etc/letsencrypt - - ./data/certbot/www:/var/www/certbot diff --git a/deploy/env.example b/deploy/env.example deleted file mode 100644 index 649bb87..0000000 --- a/deploy/env.example +++ /dev/null @@ -1 +0,0 @@ -PRETALX_TOKEN=pretalx_token diff --git a/deploy/hosts.ini b/deploy/hosts.ini deleted file mode 100644 index 9154b04..0000000 --- a/deploy/hosts.ini +++ /dev/null @@ -1,2 +0,0 @@ -[hetzner] -49.13.23.199 ansible_user=root domain_name=programapi24.europython.eu ansible_ssh_common_args='-o StrictHostKeyChecking=no' diff --git a/deploy/init-letsencrypt.sh.j2 b/deploy/init-letsencrypt.sh.j2 deleted file mode 100644 index 17fe7e2..0000000 --- a/deploy/init-letsencrypt.sh.j2 +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash - -if ! [ -x "$(command -v docker)" ]; then - echo 'Error: docker is not installed.' >&2 - exit 1 -fi - -domains=({{ domain_name }} www.{{ domain_name }}) -rsa_key_size=4096 -data_path="./data/certbot" -email="czepiel.artur@gmail.com" # Adding a valid address is strongly recommended -staging=0 # Set to 1 if you're testing your setup to avoid hitting request limits - -if [ -d "$data_path" ]; then - read -p "Existing data found for $domains. Continue and replace existing certificate? (y/N) " decision - if [ "$decision" != "Y" ] && [ "$decision" != "y" ]; then - exit - fi -fi - - -if [ ! -e "$data_path/conf/options-ssl-nginx.conf" ] || [ ! -e "$data_path/conf/ssl-dhparams.pem" ]; then - echo "### Downloading recommended TLS parameters ..." - mkdir -p "$data_path/conf" - curl -s https://raw.githubusercontent.com/certbot/certbot/master/certbot-nginx/certbot_nginx/_internal/tls_configs/options-ssl-nginx.conf > "$data_path/conf/options-ssl-nginx.conf" - curl -s https://raw.githubusercontent.com/certbot/certbot/master/certbot/certbot/ssl-dhparams.pem > "$data_path/conf/ssl-dhparams.pem" - echo -fi - -echo "### Creating dummy certificate for $domains ..." -path="/etc/letsencrypt/live/$domains" -mkdir -p "$data_path/conf/live/$domains" -docker compose run --rm --entrypoint "\ - openssl req -x509 -nodes -newkey rsa:$rsa_key_size -days 1\ - -keyout '$path/privkey.pem' \ - -out '$path/fullchain.pem' \ - -subj '/CN=localhost'" certbot -echo - - -echo "### Starting nginx ..." -docker compose up --force-recreate -d nginx -echo - -echo "### Deleting dummy certificate for $domains ..." -docker compose run --rm --entrypoint "\ - rm -Rf /etc/letsencrypt/live/$domains && \ - rm -Rf /etc/letsencrypt/archive/$domains && \ - rm -Rf /etc/letsencrypt/renewal/$domains.conf" certbot -echo - - -echo "### Requesting Let's Encrypt certificate for $domains ..." -#Join $domains to -d args -domain_args="" -for domain in "${domains[@]}"; do - domain_args="$domain_args -d $domain" -done - -# Select appropriate email arg -case "$email" in - "") email_arg="--register-unsafely-without-email" ;; - *) email_arg="--email $email" ;; -esac - -# Enable staging mode if needed -if [ $staging != "0" ]; then staging_arg="--staging"; fi - -docker compose run --rm --entrypoint "\ - certbot certonly --webroot -w /var/www/certbot \ - $staging_arg \ - $email_arg \ - $domain_args \ - --rsa-key-size $rsa_key_size \ - --agree-tos \ - --force-renewal" certbot -echo - -echo "### Reloading nginx ..." -docker compose exec nginx nginx -s reload diff --git a/deploy/nginx.conf.j2 b/deploy/nginx.conf.j2 deleted file mode 100644 index 77c8958..0000000 --- a/deploy/nginx.conf.j2 +++ /dev/null @@ -1,82 +0,0 @@ -user nginx; -worker_processes 1; - -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - map $remote_addr $remote_addr_anon { - ~(?P\d+\.\d+\.\d+)\. $ip.0; - ~(?P[^:]+:[^:]+): $ip::; - # IP addresses to not anonymize (such as your server) - 127.0.0.1 $remote_addr; - ::1 $remote_addr; - #w.x.y.z $remote_addr; - #a:b:c:d::e:f $remote_addr; - default 0.0.0.0; - } - - log_format anonymized '$remote_addr_anon - $remote_user [$time_local] ' - '"$request" $status $body_bytes_sent ' - '"$http_referer" "$http_user_agent" "$http_x_forwarded_for"'; - - access_log /var/log/nginx/access.log anonymized; - - keepalive_timeout 65; - sendfile on; - - - # To drop all the connections that don't use the valid host name - server { - listen 80 default_server; - listen [::]:80 default_server; - - listen 443 ssl http2 default_server; - listen [::]:443 ssl http2 default_server; - - ssl_certificate /etc/letsencrypt/live/{{ domain_name }}/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/{{ domain_name }}/privkey.pem; - - include /etc/letsencrypt/options-ssl-nginx.conf; - - server_name _; - - return 444; - } - - server { - listen 80; - server_name {{ domain_name }} www.{{ domain_name }}; - - location /.well-known/acme-challenge/ { - root /var/www/certbot; - } - - location / { - return 301 https://$server_name$request_uri; - } - } - - server { - listen 443 ssl; - server_name {{ domain_name }} www.{{ domain_name }}; - - ssl_certificate /etc/letsencrypt/live/{{ domain_name }}/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/{{ domain_name }}/privkey.pem; - - include /etc/letsencrypt/options-ssl-nginx.conf; - ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; - - location /2024 { - alias /usr/share/static/europython-2024; - } - } -} diff --git a/deploy/playbook.yml b/deploy/playbook.yml deleted file mode 100644 index 4abc0fd..0000000 --- a/deploy/playbook.yml +++ /dev/null @@ -1,95 +0,0 @@ -- name: Deploy programapi generator with Nginx and Let's Encrypt SSL certificate - hosts: all - become: yes - gather_facts: yes - - vars: - repository_url: https://github.com/EuroPython/programapi.git - app_version: "{{ commit_hash }}" - domain_name: programapi24.europython.eu - - tasks: - - name: Install Docker dependencies - apt: - name: "{{ packages }}" - state: present - update_cache: yes - vars: - packages: - - apt-transport-https - - ca-certificates - - curl - - gnupg - - lsb-release - - - name: Install Docker - block: - - name: Add Docker GPG key - apt_key: - url: https://download.docker.com/linux/ubuntu/gpg - state: present - - - name: Add Docker repository - apt_repository: - repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_lsb.codename }} stable - state: present - - - name: Install Docker - apt: - name: docker-ce - state: present - - - name: Add current user to docker group - user: - name: "{{ ansible_user }}" - groups: docker - append: yes - changed_when: false - - - name: Clone repository to specific version (to a temporary location) - git: - repo: "{{ repository_url }}" - dest: /tmp/repo - version: "{{ app_version }}" - - - name: Copy latest src from repository clone to /srv - command: cp -R /tmp/repo/src /srv - - - name: Copy Dockerfile - command: cp /tmp/repo/Dockerfile /srv - - - name: Copy requirements.txt - command: cp /tmp/repo/requirements.txt /srv - - - name: Copy Makefile - command: cp /tmp/repo/Makefile /srv - - - name: Copy docker-compose.yml to the remote server - ansible.builtin.template: - src: ./docker-compose.yml.j2 - dest: /srv/docker-compose.yml - - - name: Copy Nginx configuration file - ansible.builtin.template: - src: ./nginx.conf.j2 - dest: /srv/nginx.conf - - - name: Copy Init-Letsencrypt - ansible.builtin.template: - src: ./init-letsencrypt.sh.j2 - dest: /srv/init-letsencrypt.sh - - - name: Sync pretalx data every 5 minutes - ansible.builtin.cron: - name: Sync pretalx data - minute: "*/5" - job: "cd /srv && docker compose run app make all" - - - name: Periodically do a system prune - ansible.builtin.cron: - name: Docker system prune - minute: "*/5" - job: "docker system prune -f" - - - name: Start Docker compose - shell: "cd /srv && docker compose up -d --build --force-recreate" diff --git a/deploy/requirements.in b/deploy/requirements.in deleted file mode 100644 index 90d4055..0000000 --- a/deploy/requirements.in +++ /dev/null @@ -1 +0,0 @@ -ansible diff --git a/deploy/requirements.txt b/deploy/requirements.txt deleted file mode 100644 index f729ce8..0000000 --- a/deploy/requirements.txt +++ /dev/null @@ -1,26 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile -# -ansible==10.6.0 - # via -r deploy/requirements.in -ansible-core==2.17.6 - # via ansible -cffi==1.16.0 - # via cryptography -cryptography==42.0.7 - # via ansible-core -jinja2==3.1.4 - # via ansible-core -markupsafe==2.1.5 - # via jinja2 -packaging==24.0 - # via ansible-core -pycparser==2.22 - # via cffi -pyyaml==6.0.1 - # via ansible-core -resolvelib==1.0.1 - # via ansible-core diff --git a/src/config.py b/src/config.py index 1ee8d0b..4af16c3 100644 --- a/src/config.py +++ b/src/config.py @@ -6,9 +6,10 @@ class Config: event = "europython-2024" + event_dir_name = "ep2024" project_root = Path(__file__).resolve().parents[1] - raw_path = Path(f"{project_root}/data/raw/{event}") - public_path = Path(f"{project_root}/data/public/{event}") + raw_path = Path(f"{project_root}/data/raw/{event_dir_name}") + public_path = Path(f"{project_root}/data/public/{event_dir_name}") @classmethod def token(cls) -> str: