feat: serve markdown automatically based on Accept header

.github/workflows/test.yaml

@@ -35,6 +35,65 @@ jobs:
                     INTERCOM_APP_ID: ${{ secrets.INTERCOM_APP_ID }}
                     SEGMENT_TOKEN: ${{ secrets.SEGMENT_TOKEN }}
 
+            -   name: Install Nginx
+                run: |
+                    sudo apt-get update
+                    sudo apt-get install -y nginx
+
+            -   name: Start Docusaurus server
+                run: |
+                    nohup npx docusaurus serve --port 3000 --no-open &
+                    sleep 5
+                    curl -f http://localhost:3000 > /dev/null
+
+            -   name: Start Nginx with project config
+                run: |
+                    cat > default.conf <<EOF
+                    worker_processes auto;
+                    error_log $(pwd)/logs/error.log;
+                    pid $(pwd)/logs/nginx.pid;
+                    events {}
+                    http {
+                        access_log $(pwd)/logs/access.log;
+                        include $(pwd)/nginx.conf;
+                    }
+                    EOF
+                    mkdir -p $(pwd)/logs
+                    nginx -c $(pwd)/default.conf
+                    sleep 1
+
+            -   name: Run header assertions
+                run: |
+                    set -euo pipefail
+                    function assert_header() {
+                        url=$1
+                        header=$2
+                        expected=$3
+                        shift 3
+                        extra_args=("$@")
+                        actual=$(curl -s -D - -o /dev/null "${extra_args[@]}" "$url" | grep -i "^$header" | tr -d '\r' || true)
+                        echo "→ $url → $actual"
+                        echo "$actual" | grep -q "$expected" || (echo "❌ Expected '$expected' in '$header' for $url" && exit 1)
+                    }
+
+                    echo "🧪 Checking Nginx responses..."
+
+                    assert_header "http://localhost:8080/" "Content-Type" "text/html"
+                    assert_header "http://localhost:8080/" "Content-Type" "text/markdown" -H "Accept: text/markdown"
+                    assert_header "http://localhost:8080/platform/proxy/usage" "Content-Type" "text/html"
+                    assert_header "http://localhost:8080/platform/proxy/usage.md" "Content-Type" "text/markdown"
+                    assert_header "http://localhost:8080/platform/proxy/usage" "Content-Type" "text/markdown" -H "Accept: text/markdown"
+                    assert_header "http://localhost:8080/img/docs-og.png" "Content-Type" "image/png"
+                    assert_header "http://localhost:8080/img/javascript-40x40.svg" "Content-Type" "image/svg"
+                    assert_header "http://localhost:8080/llms.txt" "Content-Type" "text/markdown"
+                    assert_header "http://localhost:8080/llms-full.txt" "Content-Type" "text/markdown"
+
+                    echo "✅ All Nginx header checks passed."
+
+            -   name: Stop Nginx
+                if: always()
+                run: nginx -c $(pwd)/default.conf -s stop
+
     lint_content:
         name: Lint markdown content
         runs-on: ubuntu-latest

CONTRIBUTING.md

@@ -70,7 +70,7 @@ This will be enough to work on Platform, Academy and OpenAPI. If you want to wor
 5. Add a record to `/etc/hosts`, which maps the `docs.apify.loc` to a localhost:
 
     ```text
-    127.0.01 docs.apify.loc
+    127.0.0.1 docs.apify.loc
     ```
 
 You should be able to open https://docs.apify.loc in your browser and run all the repositories jointly as one project.

nginx.conf

@@ -1,10 +1,51 @@
+map $http_accept $wants_markdown {
+  default 0;
+  "~*text/markdown" 1;
+  "~*text/plain"    1;
+}
+
 server {
   listen 0.0.0.0:8080;
   server_name 'docs.apify.com';
 
+  # homepage: only rewrite to internal proxy when client wants markdown
+  location = / {
+    if ($wants_markdown) {
+      # use 'last' to force a new location lookup so /__proxy/ handler runs
+      rewrite ^ /__proxy/llms.txt last;
+    }
+    proxy_pass http://localhost:3000;
+  }
+
+  # direct llms.txt files, serve with markdown content type
+  location ~ ^/llms(-full)?\.txt$ {
+    proxy_pass http://localhost:3000;
+    proxy_hide_header Content-Type;
+    add_header Content-Type "text/markdown; charset=utf-8" always;
+  }
+
+  # generic docs fallback: rewrite clean URLs to .md when client wants markdown
   location / {
-    proxy_pass https://apify.github.io/apify-docs/;
+    if ($wants_markdown) {
+      # rewrite to internal proxy path; use last so new location is matched
+      rewrite ^(/[^.]+)$ /__proxy$1.md last;
+    }
+    proxy_pass http://localhost:3000;
+  }
+
+  # internal proxy handler that actually forces the markdown header
+  location ^~ /__proxy/ {
+    internal;
+
+    # strip the /__proxy prefix so upstream sees the real path
+    rewrite ^/__proxy(/.*)$ $1 break;
+
+    proxy_pass http://localhost:3000;
+    proxy_hide_header Content-Type;
+    add_header Content-Type "text/markdown; charset=utf-8" always;
   }
+
+  # proxies to other repositories
   location /api/client/js {
     proxy_pass https://apify.github.io/apify-client-js/;
   }