feat: serve markdown automatically based on Accept header (#1997)

Alters the nginx config so it checks the `Accept` header, if it contains
`text/plain` or `text/markdown`, we serve the markdown version directly.
Also added tests for this behavior to the existing PR check.

The previews are likely using the config from master, so there it
doesn't work yet (it should after we merge this).

Since the test workflow now effectively verifies the nginx config is
valid, I guess we can drop the `nginx.conf-test.yaml` workflow.

Author: B4nan

.github/workflows/nginx.conf-test.yaml

@@ -35,6 +35,65 @@ jobs:
                     INTERCOM_APP_ID: ${{ secrets.INTERCOM_APP_ID }}
                     SEGMENT_TOKEN: ${{ secrets.SEGMENT_TOKEN }}
 
+            -   name: Install Nginx
+                run: |
+                    sudo apt-get update
+                    sudo apt-get install -y nginx
+
+            -   name: Start Docusaurus server
+                run: |
+                    nohup npx docusaurus serve --port 3000 --no-open &
+                    sleep 5
+                    curl -f http://localhost:3000 > /dev/null
+
+            -   name: Start Nginx with project config
+                run: |
+                    cat > default.conf <<EOF
+                    worker_processes auto;
+                    error_log $(pwd)/logs/error.log;
+                    pid $(pwd)/logs/nginx.pid;
+                    events {}
+                    http {
+                        access_log $(pwd)/logs/access.log;
+                        include $(pwd)/nginx.conf;
+                    }
+                    EOF
+                    mkdir -p $(pwd)/logs
+                    nginx -c $(pwd)/default.conf
+                    sleep 1
+
+            -   name: Run header assertions
+                run: |
+                    set -euo pipefail
+                    function assert_header() {
+                        url=$1
+                        header=$2
+                        expected=$3
+                        shift 3
+                        extra_args=("$@")
+                        actual=$(curl -s -D - -o /dev/null "${extra_args[@]}" "$url" | grep -i "^$header" | tr -d '\r' || true)
+                        echo "→ $url → $actual"
+                        echo "$actual" | grep -q "$expected" || (echo "❌ Expected '$expected' in '$header' for $url" && exit 1)
+                    }
+
+                    echo "🧪 Checking Nginx responses..."
+
+                    assert_header "http://localhost:8080/" "Content-Type" "text/html"
+                    assert_header "http://localhost:8080/" "Content-Type" "text/markdown" -H "Accept: text/markdown"
+                    assert_header "http://localhost:8080/platform/proxy/usage" "Content-Type" "text/html"
+                    assert_header "http://localhost:8080/platform/proxy/usage.md" "Content-Type" "text/markdown"
+                    assert_header "http://localhost:8080/platform/proxy/usage" "Content-Type" "text/markdown" -H "Accept: text/markdown"
+                    assert_header "http://localhost:8080/img/docs-og.png" "Content-Type" "image/png"
+                    assert_header "http://localhost:8080/img/javascript-40x40.svg" "Content-Type" "image/svg"
+                    assert_header "http://localhost:8080/llms.txt" "Content-Type" "text/markdown"
+                    assert_header "http://localhost:8080/llms-full.txt" "Content-Type" "text/markdown"
+
+                    echo "✅ All Nginx header checks passed."
+
+            -   name: Stop Nginx
+                if: always()
+                run: nginx -c $(pwd)/default.conf -s stop
+
     lint_content:
         name: Lint markdown content
         runs-on: ubuntu-latest

.github/workflows/test.yaml

@@ -70,7 +70,7 @@ This will be enough to work on Platform, Academy and OpenAPI. If you want to wor
 5. Add a record to `/etc/hosts`, which maps the `docs.apify.loc` to a localhost:
 
     ```text
-    127.0.01 docs.apify.loc
+    127.0.0.1 docs.apify.loc
     ```
 
 You should be able to open https://docs.apify.loc in your browser and run all the repositories jointly as one project.

CONTRIBUTING.md

@@ -1,10 +1,51 @@
+map $http_accept $wants_markdown {
+  default 0;
+  "~*text/markdown" 1;
+  "~*text/plain"    1;
+}
+
 server {
   listen 0.0.0.0:8080;
   server_name 'docs.apify.com';
 
+  # homepage: only rewrite to internal proxy when client wants markdown
+  location = / {
+    if ($wants_markdown) {
+      # use 'last' to force a new location lookup so /__proxy/ handler runs
+      rewrite ^ /__proxy/llms.txt last;
+    }
+    proxy_pass http://localhost:3000;
+  }
+
+  # direct llms.txt files, serve with markdown content type
+  location ~ ^/llms(-full)?\.txt$ {
+    proxy_pass http://localhost:3000;
+    proxy_hide_header Content-Type;
+    add_header Content-Type "text/markdown; charset=utf-8" always;
+  }
+
+  # generic docs fallback: rewrite clean URLs to .md when client wants markdown
   location / {
-    proxy_pass https://apify.github.io/apify-docs/;
+    if ($wants_markdown) {
+      # rewrite to internal proxy path; use last so new location is matched
+      rewrite ^(/[^.]+)$ /__proxy$1.md last;
+    }
+    proxy_pass http://localhost:3000;
+  }
+
+  # internal proxy handler that actually forces the markdown header
+  location ^~ /__proxy/ {
+    internal;
+
+    # strip the /__proxy prefix so upstream sees the real path
+    rewrite ^/__proxy(/.*)$ $1 break;
+
+    proxy_pass http://localhost:3000;
+    proxy_hide_header Content-Type;
+    add_header Content-Type "text/markdown; charset=utf-8" always;
   }
+
+  # proxies to other repositories
   location /api/client/js {
     proxy_pass https://apify.github.io/apify-client-js/;
   }