Pull from upstream/main and resolve conflicts.

Author: RohanJnr

.flake8

@@ -3,7 +3,7 @@ max-line-length=100
 docstring-convention=all
 import-order-style=pycharm
 application_import_names=pydis_site
-exclude=__pycache__, venv, .venv, **/migrations/**, .cache/
+exclude=__pycache__, venv, .venv, **/migrations/**, .cache/, gunicorn.conf.py
 ignore=
     B311,W503,E226,S311,T000
     # Missing Docstrings

.github/PULL_REQUEST_TEMPLATE/pull_request.md

@@ -0,0 +1,22 @@
+## Summary
+<!-- Please provide a brief description of this PR along with the related issue it fixes -->
+
+
+## Description of changes
+<!-- Describe what changes you've made, and how you implemented them -->
+
+
+### I confirm I have:
+<!-- Replace [ ] below with [X] to check the boxes -->
+- [ ] Joined the [Python Discord community](discord.gg/python)
+- [ ] Read the [Code of Conduct](https://www.pydis.com/pages/code-of-conduct) and agree to it
+- [ ] I have discussed implementing this feature on the relevant service (Discord, GitHub, etc.)
+
+
+### I have changed API models and I ensure I have:
+<!-- Please remove this section if you haven't edited files under pydis_site/apps/api/models -->
+- [ ] Opened a PR updating the model on the [API GitHub Repository](https://github.com/python-discord/api)
+
+**OR**
+
+- [ ] Opened an issue on the [API GitHub Repository](https://github.com/python-discord/api) explaining what changes need to be made

.github/workflows/deploy.yaml

@@ -41,7 +41,7 @@ jobs:
         uses: Azure/k8s-deploy@v1
         with:
           manifests: |
-              site/deployment.yaml
+              namespaces/default/site/deployment.yaml
           images: 'ghcr.io/python-discord/site:${{ steps.sha_tag.outputs.tag }}'
           kubectl-version: 'latest'
 

.github/workflows/static-preview.yaml

@@ -0,0 +1,77 @@
+name: Build & Publish Static Preview
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  build:
+    name: Build Static Preview
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+     # Create a commit SHA-based tag for the container repositories
+      - name: Create SHA Container Tag
+        id: sha_tag
+        run: |
+          tag=$(cut -c 1-7 <<< $GITHUB_SHA)
+          echo "::set-output name=tag::$tag"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Github Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Build the container, including an inline cache manifest to
+      # allow us to use the registry as a cache source.
+      - name: Build Docker Image (Main)
+        uses: docker/build-push-action@v2
+        if: github.ref == 'refs/heads/main'
+        with:
+          context: .
+          push: true
+          cache-from: type=registry,ref=ghcr.io/python-discord/static-site:latest
+          cache-to: type=inline
+          tags: |
+            ghcr.io/python-discord/static-site:latest
+            ghcr.io/python-discord/static-site:${{ steps.sha_tag.outputs.tag }}
+          build-args: |
+            git_sha=${{ github.sha }}
+            STATIC_BUILD=TRUE
+
+      - name: Extract Build From Docker Image (Main)
+        if: github.ref == 'refs/heads/main'
+        run: |
+          mkdir docker_build \
+          && docker run --entrypoint /bin/echo --name site \
+          ghcr.io/python-discord/static-site:${{ steps.sha_tag.outputs.tag }} \
+          && docker cp site:/app docker_build/
+
+      # Build directly to a local folder
+      - name: Build Docker Image (PR)
+        uses: docker/build-push-action@v2
+        if: github.ref != 'refs/heads/main'
+        with:
+          context: .
+          push: false
+          cache-from: type=registry,ref=ghcr.io/python-discord/static-site:latest
+          outputs: type=local,dest=docker_build/
+          build-args: |
+            git_sha=${{ github.sha }}
+            STATIC_BUILD=TRUE
+
+      - name: Upload Build
+        uses: actions/upload-artifact@v2
+        with:
+          name: static-build
+          path: docker_build/app/build/
+          if-no-files-found: error

.gitignore

@@ -126,3 +126,6 @@ staticfiles/
 
 *.js.tmp
 log.*
+
+# Local Netlify folder
+.netlify

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9.5-slim-buster
+FROM --platform=linux/amd64 python:3.9-slim-buster
 
 # Allow service to handle stops gracefully
 STOPSIGNAL SIGQUIT
@@ -24,6 +24,24 @@ ENV GIT_SHA=$git_sha
 # Copy the source code in last to optimize rebuilding the image
 COPY . .
 
+# Set dummy variables so collectstatic can load settings.py
+RUN \
+    # Set BUILDING_DOCKER to anything but undefined so settings.py
+    # does not insert django_prometheus into the list of installed apps.
+    # This prevents django_prometheus from attempting to connect to the database
+    # when the collectstatic task is ran.
+    BUILDING_DOCKER=yes \
+    SECRET_KEY=dummy_value \
+    DATABASE_URL=postgres://localhost \
+    METRICITY_DB_URL=postgres://localhost \
+    python manage.py collectstatic --noinput --clear
+
+# Build static files if we are doing a static build
+ARG STATIC_BUILD=false
+RUN if [ $STATIC_BUILD = "TRUE" ] ; \
+  then SECRET_KEY=dummy_value python manage.py distill-local build --traceback --force ; \
+fi
+
 # Run web server through custom manager
 ENTRYPOINT ["python", "manage.py"]
 CMD ["run"]

README.md

@@ -12,7 +12,7 @@ This is all of the code that is responsible for maintaining [our website][9] and
 The website is built on Django and should be simple to set up and get started with.
 If you happen to run into issues with setup, please don't hesitate to open an issue!
 
-If you're looking to contribute or play around with the code, take a look at [the wiki][10] or the [`docs` directory](docs). If you're looking for things to do, check out [our issues][11].
+If you're looking to contribute or play around with the code, take a look at [the wiki][10]. If you're looking for things to do, check out [our issues][11].
 
 [1]: https://github.com/python-discord/site/workflows/Lint%20&%20Test/badge.svg?branch=main
 [2]: https://github.com/python-discord/site/actions?query=workflow%3A%22Lint+%26+Test%22+branch%3Amain

docker-compose.yml

@@ -18,6 +18,11 @@ services:
       POSTGRES_DB: pysite
       POSTGRES_PASSWORD: pysite
       POSTGRES_USER: pysite
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U pysite"]
+      interval: 2s
+      timeout: 1s
+      retries: 5
     volumes:
       - ./postgres/init.sql:/docker-entrypoint-initdb.d/init.sql
 
@@ -35,7 +40,8 @@ services:
     ports:
       - "127.0.0.1:8000:8000"
     depends_on:
-      - postgres
+      postgres:
+        condition: service_healthy
     tty: true
     volumes:
       - .:/app:ro

gunicorn.conf.py

@@ -0,0 +1,10 @@
+"""
+Configuration file for gunicorn.
+
+Code taken from https://github.com/prometheus/client_python#multiprocess-mode-eg-gunicorn
+"""
+from prometheus_client import multiprocess
+
+
+def child_exit(server, worker) -> None:
+    multiprocess.mark_process_dead(worker.pid)

manage.py

@@ -1,10 +1,8 @@
 #!/usr/bin/env python
 import os
-import re
-import socket
+import platform
 import sys
-import time
-from typing import List
+from pathlib import Path
 
 import django
 from django.contrib.auth import get_user_model
@@ -42,7 +40,7 @@ class SiteManager:
         --verbose  Sets verbose console output.
     """
 
-    def __init__(self, args: List[str]):
+    def __init__(self, args: list[str]):
         self.debug = "--debug" in args
         self.silent = "--silent" in args
 
@@ -84,38 +82,6 @@ def create_superuser() -> None:
         else:
             print(f"Existing bot token found: {token}")
 
-    @staticmethod
-    def wait_for_postgres() -> None:
-        """Wait for the PostgreSQL database specified in DATABASE_URL."""
-        print("Waiting for PostgreSQL database.")
-
-        # Get database URL based on environmental variable passed in compose
-        database_url = os.environ["DATABASE_URL"]
-        match = re.search(r"@([\w.]+):(\d+)/", database_url)
-        if not match:
-            raise OSError("Valid DATABASE_URL environmental variable not found.")
-        domain = match.group(1)
-        port = int(match.group(2))
-
-        # Attempt to connect to the database socket
-        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-
-        attempts_left = 10
-        while attempts_left:
-            try:
-                # Ignore 'incomplete startup packet'
-                s.connect((domain, port))
-                s.shutdown(socket.SHUT_RDWR)
-                print("Database is ready.")
-                break
-            except socket.error:
-                attempts_left -= 1
-                print("Not ready yet, retrying.")
-                time.sleep(0.5)
-        else:
-            print("Database could not be found, exiting.")
-            sys.exit(1)
-
     @staticmethod
     def set_dev_site_name() -> None:
         """Set the development site domain in admin from default example."""
@@ -133,15 +99,19 @@ def prepare_server(self) -> None:
         """Perform preparation tasks before running the server."""
         django.setup()
 
-        if self.debug:
-            self.wait_for_postgres()
-
         print("Applying migrations.")
         call_command("migrate", verbosity=self.verbosity)
-        print("Collecting static files.")
-        call_command("collectstatic", interactive=False, clear=True, verbosity=self.verbosity)
 
         if self.debug:
+            # In Production, collectstatic is ran in the Docker image
+            print("Collecting static files.")
+            call_command(
+                "collectstatic",
+                interactive=False,
+                clear=True,
+                verbosity=self.verbosity - 1
+            )
+
             self.set_dev_site_name()
             self.create_superuser()
 
@@ -169,18 +139,32 @@ def run_server(self) -> None:
             "--preload",
             "-b", "0.0.0.0:8000",
             "pydis_site.wsgi:application",
-            "--threads", "8",
             "-w", "2",
-            "--max-requests", "1000",
-            "--max-requests-jitter", "50",
             "--statsd-host", "graphite.default.svc.cluster.local:8125",
             "--statsd-prefix", "site",
+            "--config", "file:gunicorn.conf.py"
         ]
 
         # Run gunicorn for the production server.
         gunicorn.app.wsgiapp.run()
 
 
+def clean_up_static_files(build_folder: Path) -> None:
+    """Recursively loop over the build directory and fix links."""
+    for file in build_folder.iterdir():
+        if file.is_dir():
+            clean_up_static_files(file)
+        elif file.name.endswith(".html"):
+            # Fix parent host url
+            new = file.read_text(encoding="utf-8").replace(f"//{os.getenv('PARENT_HOST')}", "")
+
+            # Fix windows paths if on windows
+            if platform.system() == "Windows":
+                new = new.replace("%5C", "/")
+
+            file.write_text(new, encoding="utf-8")
+
+
 def main() -> None:
     """Entry point for Django management script."""
     # Use the custom site manager for launching the server
@@ -189,8 +173,23 @@ def main() -> None:
 
     # Pass any others directly to standard management commands
     else:
+        _static_build = "distill" in sys.argv[1]
+
+        if _static_build:
+            # Build a static version of the site with no databases and API support
+            os.environ["STATIC_BUILD"] = "True"
+            if not os.getenv("PARENT_HOST"):
+                os.environ["PARENT_HOST"] = "REPLACE_THIS.HOST"
+
         execute_from_command_line(sys.argv)
 
+        if _static_build:
+            # Clean up parent host in generated files
+            for arg in sys.argv[2:]:
+                if not arg.startswith("-"):
+                    clean_up_static_files(Path(arg))
+                    break
+
 
 if __name__ == '__main__':
     main()