feat: serve markdown automatically based on Accept header

.github/workflows/test.yaml

@@ -35,65 +35,65 @@ jobs:
                     INTERCOM_APP_ID: ${{ secrets.INTERCOM_APP_ID }}
                     SEGMENT_TOKEN: ${{ secrets.SEGMENT_TOKEN }}
 
-#            -   name: Install Nginx
-#                run: |
-#                    sudo apt-get update
-#                    sudo apt-get install -y nginx
-#
-#            -   name: Start Docusaurus server
-#                run: |
-#                    nohup npx docusaurus serve --port 3000 --no-open &
-#                    sleep 5
-#                    curl -f http://localhost:3000 > /dev/null
-#
-#            -   name: Start Nginx with project config
-#                run: |
-#                    cat > default.conf <<EOF
-#                    worker_processes auto;
-#                    error_log $(pwd)/logs/error.log;
-#                    pid $(pwd)/logs/nginx.pid;
-#                    events {}
-#                    http {
-#                        access_log $(pwd)/logs/access.log;
-#                        include $(pwd)/nginx.conf;
-#                    }
-#                    EOF
-#                    sed -i 's|https://apify.github.io/apify-docs|http://localhost:3000|g' default.conf
-#                    mkdir -p $(pwd)/logs
-#                    nginx -c $(pwd)/default.conf
-#                    sleep 1
-#
-#            -   name: Run header assertions
-#                run: |
-#                    set -euo pipefail
-#                    function assert_header() {
-#                        url=$1
-#                        header=$2
-#                        expected=$3
-#                        shift 3
-#                        extra_args=("$@")
-#                        actual=$(curl -s -D - -o /dev/null "${extra_args[@]}" "$url" | grep -i "^$header" | tr -d '\r' || true)
-#                        echo "→ $url → $actual"
-#                        echo "$actual" | grep -q "$expected" || (echo "❌ Expected '$expected' in '$header' for $url" && exit 1)
-#                    }
-#
-#                    echo "🧪 Checking Nginx responses..."
-#
-#                    assert_header "http://localhost:8080/" "Content-Type" "text/html"
-#                    assert_header "http://localhost:8080/" "Content-Type" "text/markdown" -H "Accept: text/markdown"
-#                    assert_header "http://localhost:8080/platform/proxy/usage" "Content-Type" "text/html"
-#                    assert_header "http://localhost:8080/platform/proxy/usage.md" "Content-Type" "text/markdown"
-#                    assert_header "http://localhost:8080/platform/proxy/usage" "Content-Type" "text/markdown" -H "Accept: text/markdown"
-#                    assert_header "http://localhost:8080/img/docs-og.png" "Content-Type" "image/png"
-#                    assert_header "http://localhost:8080/img/javascript-40x40.svg" "Content-Type" "image/svg"
-#                    assert_header "http://localhost:8080/llms.txt" "Content-Type" "text/markdown"
-#                    assert_header "http://localhost:8080/llms-full.txt" "Content-Type" "text/markdown"
-#
-#                    echo "✅ All Nginx header checks passed."
-#
-#            -   name: Stop Nginx
-#                if: always()
-#                run: nginx -c $(pwd)/default.conf -s stop
+            -   name: Install Nginx
+                run: |
+                    sudo apt-get update
+                    sudo apt-get install -y nginx
+
+            -   name: Start Docusaurus server
+                run: |
+                    nohup npx docusaurus serve --port 3000 --no-open &
+                    sleep 5
+                    curl -f http://localhost:3000 > /dev/null
+
+            -   name: Start Nginx with project config
+                run: |
+                    cat > default.conf <<EOF
+                    worker_processes auto;
+                    error_log $(pwd)/logs/error.log;
+                    pid $(pwd)/logs/nginx.pid;
+                    events {}
+                    http {
+                        access_log $(pwd)/logs/access.log;
+                        include $(pwd)/nginx.conf;
+                    }
+                    EOF
+                    sed -i 's|https://apify.github.io/apify-docs|http://localhost:3000|g' default.conf
+                    mkdir -p $(pwd)/logs
+                    nginx -c $(pwd)/default.conf
+                    sleep 1
+
+            -   name: Run header assertions
+                run: |
+                    set -euo pipefail
+                    function assert_header() {
+                        url=$1
+                        header=$2
+                        expected=$3
+                        shift 3
+                        extra_args=("$@")
+                        actual=$(curl -s -D - -o /dev/null "${extra_args[@]}" "$url" | grep -i "^$header" | tr -d '\r' || true)
+                        echo "→ $url → $actual"
+                        echo "$actual" | grep -q "$expected" || (echo "❌ Expected '$expected' in '$header' for $url" && exit 1)
+                    }
+
+                    echo "🧪 Checking Nginx responses..."
+
+                    assert_header "http://localhost:8080/" "Content-Type" "text/html"
+                    assert_header "http://localhost:8080/" "Content-Type" "text/markdown" -H "Accept: text/markdown"
+                    assert_header "http://localhost:8080/platform/proxy/usage" "Content-Type" "text/html"
+                    assert_header "http://localhost:8080/platform/proxy/usage.md" "Content-Type" "text/markdown"
+                    assert_header "http://localhost:8080/platform/proxy/usage" "Content-Type" "text/markdown" -H "Accept: text/markdown"
+                    assert_header "http://localhost:8080/img/docs-og.png" "Content-Type" "image/png"
+                    assert_header "http://localhost:8080/img/javascript-40x40.svg" "Content-Type" "image/svg"
+                    assert_header "http://localhost:8080/llms.txt" "Content-Type" "text/markdown"
+                    assert_header "http://localhost:8080/llms-full.txt" "Content-Type" "text/markdown"
+
+                    echo "✅ All Nginx header checks passed."
+
+            -   name: Stop Nginx
+                if: always()
+                run: nginx -c $(pwd)/default.conf -s stop
 
     lint_content:
         name: Lint markdown content

nginx.conf

@@ -1,9 +1,47 @@
+map $http_accept $serve_markdown {
+    default         0;
+    ~*text/plain    1;
+    ~*text/markdown 1;
+}
+
+map $request_uri $has_no_extension {
+    ~^[^.]*$ 1;
+    default  0;
+}
+
 server {
     listen 0.0.0.0:8080;
-    server_name docs.apify.com;
+    server_name docs.apify.com docs.apify.loc;
+
+    proxy_set_header X-Real-IP       $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+
+    # comment out the resolver and use localhost:3000 for local development
+    set $backend "https://apify.github.io/apify-docs";
+    resolver 1.1.1.1 8.8.8.8 valid=30s ipv6=off;
+
+    location = / {
+        if ($serve_markdown) {
+            rewrite ^ /llms.txt last;
+        }
+        proxy_pass $backend/;
+    }
+
+    location ~ ^/(llms|llms-full)\.txt$ {
+        proxy_hide_header Content-Type;
+        add_header Content-Type 'text/markdown; charset=utf-8' always;
+        proxy_pass $backend$uri;
+    }
 
     location / {
-        proxy_pass https://apify.github.io/apify-docs/;
+        set $rewrite_condition "$serve_markdown$has_no_extension";
+        set $proxy_path $request_uri;
+
+        if ($rewrite_condition = "11") {
+            set $proxy_path "${request_uri}.md";
+        }
+        proxy_pass $backend$proxy_path;
     }
 
     # proxies to other repositories