Merge pull request #23 from wikimediabrasil/video-download-toolforge

Video download feature

Author: arcstur

.github/workflows/ci.yml

@@ -24,16 +24,19 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
+    - name: Install system dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y librsvg2-bin ffmpeg
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r src/requirements.txt
-        pip install -r src/requirements-dev.txt
+        pip install -r requirements.txt
+        pip install -r requirements-dev.txt
 
     - name: Run tests
       env:
         SECRET_KEY: dummysecretkey
         DEBUG: True
-      run: |
-        cd src
-        python3 manage.py test
+      run: python3 manage.py test

.github/workflows/toolforge-deploy.yml

@@ -23,18 +23,13 @@ jobs:
           key: ${{ secrets.SSH_PRIVATE_KEY }}
           port: ${{ vars.SSH_PORT }}
           request_pty: true
+          command_timeout: 30m
           script: |
             set -xe
-            become infographics git -C ./www/python/src pull origin main
-            become infographics webservice node18 shell -- npm --prefix ./www/python/client/ install
-            become infographics webservice --mem 1Gi node18 shell -- npm --prefix ./www/python/client/ run build
-            become infographics mkdir -p ./www/python/src/web/templates
-            become infographics mkdir -p ./www/python/src/web/static/frontend
-            become infographics cp ./www/python/client/dist/index.html ./www/python/src/web/templates/index.html
-            become infographics rm -rf ./www/python/src/web/static/frontend/assets
-            become infographics cp -r ./www/python/client/dist/assets ./www/python/src/web/static/frontend/assets
-            become infographics webservice python3.11 shell -- webservice-python-bootstrap
-            become infographics webservice python3.11 shell -- ./www/python/venv/bin/python ./www/python/src/manage.py migrate
-            become infographics webservice python3.11 shell -- ./www/python/venv/bin/python ./www/python/src/manage.py collectstatic --noinput --clear
-            become infographics webservice python3.11 restart
-            become infographics webservice python3.11 status
+            become infographics toolforge envvars create DEBUG "True"
+            become infographics toolforge envvars create MEDIA_ROOT "/data/project/infographics/media"
+            become infographics toolforge build clean -y
+            become infographics toolforge build start https://github.com/wikimediabrasil/wiki_infographics
+            become infographics toolforge jobs run --image tool-infographics/tool-infographics:latest --command "migrate" --no-filelog --wait --mount=all migrate
+            become infographics toolforge webservice stop
+            become infographics toolforge webservice buildservice start --mount all --mem 3Gi --cpu 2

.gitignore

@@ -1,9 +1,12 @@
 # Django collected static files
-/src/static
+/static
+
+# Generated files
+/media
 
 # Static files and templates from frontend code
-/src/web/static/frontend
-/src/web/templates/index.html
+/web/templates/index.html
+/web/static/frontend/assets
 
 .DS_*
 *.log

Aptfile

@@ -0,0 +1,2 @@
+ffmpeg
+librsvg2-bin

Dockerfile

@@ -0,0 +1,2 @@
+web: gunicorn --bind=0.0.0.0:8000 --workers=4 --timeout 120 --log-level debug --forwarded-allow-ips=* infographics.wsgi:application
+migrate: python manage.py migrate

Procfile

@@ -6,16 +6,64 @@ The tool is composed of a Django backend and a React frontend, both on this repo
 
 ## Running locally
 
-To run the tool locally, you need Docker and Docker Compose.
+To run the tool locally, have NodeJS, Python, `rsvg-convert` and `ffmpeg` installed. You need two terminals.
+
+In one of them, run the frontend
 
 ```bash
-docker-compose up --build
+npm install
+npm run dev
+```
+
+And open your browser at <http://localhost:5173/web/infographics/>.
+
+For the backend, run
+
+```bash
+python3 manage.py runserver
 ```
 
 Open the app at <http://localhost:7840>.
 
 If you make any changes, you can kill the process and run again, or restart the app container with `docker-compose up -d --build --force-recreate app`.
 
+
 ## License
 
 This project is licensed under the [MIT License](https://opensource.org/license/mit) - see the LICENSE file for details.
+
+## Toolforge deployment
+
+Since we need apt packages we are using [Toolforge's custom builds feature](https://wikitech.wikimedia.org/wiki/Help:Toolforge/Building_container_images) so that we can have `ffmpeg` and `rsvg-convert` utilities installed.
+
+On toolforge, run the following to clean build space, build the image, run migrations and start the server.
+
+```bash
+toolforge build clean -y
+toolforge build start https://github.com/wikimediabrasil/wiki_infographics
+toolforge jobs run --image tool-infographics/tool-infographics:latest --command "migrate" --wait --mount=all migrate
+toolforge webservice buildservice restart --mount all
+```
+
+We are using `--mount=all` because we're still using SQLite and we need a directory to store the videos.
+
+Set environment variables with `toolforge envvars create <VARIABLE> <VALUE>`.
+
+## Running locally with buildpack
+
+This is the way to build the image locally:
+
+```bash
+sudo pack build --builder tools-harbor.wmcloud.org/toolforge/heroku-builder:22 --buildpack heroku/nodejs --buildpack heroku/python --buildpack heroku/procfile --buildpack fagiani/apt infographics
+sudo docker run -e PORT=8000 -e DEBUG=True -e SECRET_KEY=123 -p 8000:8000 -it  --entrypoint 'bash' infographics
+```
+
+Inside the container, you can run `migrate` or `web`. The binaries are not directly available to the path, these fixes are necessary when entering bash:
+
+```bash
+source /layers/fagiani_apt/apt/.profile.d/000_apt.sh
+cp -r /layers/fagiani_apt/apt/usr/lib/x86_64-linux-gnu/*/* /layers/fagiani_apt/apt/usr/lib/x86_64-linux-gnu/
+```
+
+Then it is possible to run `web` and have `ffmpeg` and `rsvg-convert` installed.
+

README.md

@@ -1,6 +1,6 @@
 from django.apps import AppConfig
 
 
-class QueryConfig(AppConfig):
+class ApiConfig(AppConfig):
     default_auto_field = 'django.db.models.BigAutoField'
-    name = 'query'
+    name = 'api'