Skip to content

feat: serve markdown automatically based on Accept header for child repositories #2032

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Oct 22, 2025
Merged

feat: serve markdown automatically based on Accept header for child repositories #2032

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3b12e76
feat: serve markdown automatically based on `Accept` header for child…
B4nan Oct 21, 2025
b8ff8bb
fix content type of llms.txt
B4nan Oct 21, 2025
b09586e
fix python client rules
B4nan Oct 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 46 additions & 2 deletions .github/workflows/test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,7 @@ jobs:
echo "$actual" | grep -q "$expected" || (echo "❌ Expected '$expected' in '$header' for $url" && exit 1)
}
echo "🧪 Checking Nginx responses..."
echo "🧪 Checking Nginx responses... (apify-docs)"
assert_header "http://localhost:8080/" "Content-Type" "text/html"
assert_header "http://localhost:8080/" "Content-Type" "text/markdown" -H "Accept: text/markdown"
assert_header "http://localhost:8080/platform/proxy/usage" "Content-Type" "text/html"
Expand All @@ -89,6 +88,51 @@ jobs:
assert_header "http://localhost:8080/llms.txt" "Content-Type" "text/markdown"
assert_header "http://localhost:8080/llms-full.txt" "Content-Type" "text/markdown"
echo "🧪 Checking Nginx responses... (apify-sdk-js)"
assert_header "http://localhost:8080/sdk/js" "Content-Type" "text/html"
assert_header "http://localhost:8080/sdk/js" "Content-Type" "text/markdown" -H "Accept: text/markdown"
assert_header "http://localhost:8080/sdk/js/docs/guides/apify-platform" "Content-Type" "text/html"
assert_header "http://localhost:8080/sdk/js/docs/guides/apify-platform.md" "Content-Type" "text/markdown"
assert_header "http://localhost:8080/sdk/js/docs/guides/apify-platform" "Content-Type" "text/markdown" -H "Accept: text/markdown"
assert_header "http://localhost:8080/sdk/js/llms.txt" "Content-Type" "text/markdown"
assert_header "http://localhost:8080/sdk/js/llms-full.txt" "Content-Type" "text/markdown"
echo "🧪 Checking Nginx responses... (apify-sdk-python)"
assert_header "http://localhost:8080/sdk/python" "Content-Type" "text/html"
assert_header "http://localhost:8080/sdk/python" "Content-Type" "text/markdown" -H "Accept: text/markdown"
assert_header "http://localhost:8080/sdk/python/docs/changelog" "Content-Type" "text/html"
assert_header "http://localhost:8080/sdk/python/docs/changelog.md" "Content-Type" "text/markdown"
assert_header "http://localhost:8080/sdk/python/docs/changelog" "Content-Type" "text/markdown" -H "Accept: text/markdown"
assert_header "http://localhost:8080/sdk/python/llms.txt" "Content-Type" "text/markdown"
assert_header "http://localhost:8080/sdk/python/llms-full.txt" "Content-Type" "text/markdown"
echo "🧪 Checking Nginx responses... (apify-client-js)"
assert_header "http://localhost:8080/api/client/js" "Content-Type" "text/html"
assert_header "http://localhost:8080/api/client/js" "Content-Type" "text/markdown" -H "Accept: text/markdown"
assert_header "http://localhost:8080/api/client/js/docs/changelog" "Content-Type" "text/html"
assert_header "http://localhost:8080/api/client/js/docs/changelog.md" "Content-Type" "text/markdown"
assert_header "http://localhost:8080/api/client/js/docs/changelog" "Content-Type" "text/markdown" -H "Accept: text/markdown"
assert_header "http://localhost:8080/api/client/js/llms.txt" "Content-Type" "text/markdown"
assert_header "http://localhost:8080/api/client/js/llms-full.txt" "Content-Type" "text/markdown"
echo "🧪 Checking Nginx responses... (apify-client-python)"
assert_header "http://localhost:8080/api/client/python" "Content-Type" "text/html"
assert_header "http://localhost:8080/api/client/python" "Content-Type" "text/markdown" -H "Accept: text/markdown"
assert_header "http://localhost:8080/api/client/python/docs/changelog" "Content-Type" "text/html"
assert_header "http://localhost:8080/api/client/python/docs/changelog.md" "Content-Type" "text/markdown"
assert_header "http://localhost:8080/api/client/python/docs/changelog" "Content-Type" "text/markdown" -H "Accept: text/markdown"
assert_header "http://localhost:8080/api/client/python/llms.txt" "Content-Type" "text/markdown"
assert_header "http://localhost:8080/api/client/python/llms-full.txt" "Content-Type" "text/markdown"
echo "🧪 Checking Nginx responses... (apify-cli)"
assert_header "http://localhost:8080/cli" "Content-Type" "text/html"
assert_header "http://localhost:8080/cli" "Content-Type" "text/markdown" -H "Accept: text/markdown"
assert_header "http://localhost:8080/cli/docs/changelog" "Content-Type" "text/html"
assert_header "http://localhost:8080/cli/docs/changelog.md" "Content-Type" "text/markdown"
assert_header "http://localhost:8080/cli/docs/changelog" "Content-Type" "text/markdown" -H "Accept: text/markdown"
assert_header "http://localhost:8080/cli/llms.txt" "Content-Type" "text/markdown"
assert_header "http://localhost:8080/cli/llms-full.txt" "Content-Type" "text/markdown"
echo "✅ All Nginx header checks passed."
- name: Stop Nginx
Expand Down
225 changes: 207 additions & 18 deletions nginx.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,22 +44,211 @@ server {
proxy_pass $backend$proxy_path;
}

# proxies to other repositories
location /api/client/js {
### Repository path: "/sdk/js"

location = /sdk/js {
if ($serve_markdown) {
rewrite ^ /sdk-js-llms.txt last;
}
proxy_pass https://apify.github.io/apify-sdk-js/;
}
location = /sdk/js/ {
if ($serve_markdown) {
rewrite ^ /sdk-js-llms.txt last;
}
proxy_pass https://apify.github.io/apify-sdk-js/;
}

location = /sdk-js-llms.txt {
proxy_hide_header Content-Type;
add_header Content-Type 'text/markdown; charset=utf-8' always;
proxy_pass https://apify.github.io/apify-sdk-js/llms.txt;
}

location ~ ^/sdk/js/(llms|llms-full)\.txt$ {
rewrite ^/sdk/js(/.*)$ $1 break;
proxy_hide_header Content-Type;
add_header Content-Type 'text/markdown; charset=utf-8' always;
proxy_pass https://apify.github.io/apify-sdk-js$uri;
}

location ~ ^/sdk/js/(.*)$ {
set $path_suffix $1;
set $proxy_path "/$path_suffix";

set $rewrite_condition "$serve_markdown$has_no_extension";

if ($rewrite_condition = "11") {
set $proxy_path "${proxy_path}.md";
}

proxy_pass https://apify.github.io/apify-sdk-js$proxy_path;
}

### Repository path: "/sdk/python"

location = /sdk/python {
if ($serve_markdown) {
rewrite ^ /sdk-python-llms.txt last;
}
proxy_pass https://apify.github.io/apify-sdk-python/;
}
location = /sdk/python/ {
if ($serve_markdown) {
rewrite ^ /sdk-python-llms.txt last;
}
proxy_pass https://apify.github.io/apify-sdk-python/;
}

location = /sdk-python-llms.txt {
proxy_hide_header Content-Type;
add_header Content-Type 'text/markdown; charset=utf-8' always;
proxy_pass https://apify.github.io/apify-sdk-python/llms.txt;
}

location ~ ^/sdk/python/(llms|llms-full)\.txt$ {
rewrite ^/sdk/python(/.*)$ $1 break;
proxy_hide_header Content-Type;
add_header Content-Type 'text/markdown; charset=utf-8' always;
proxy_pass https://apify.github.io/apify-sdk-python$uri;
}

location ~ ^/sdk/python/(.*)$ {
set $path_suffix $1;
set $proxy_path "/$path_suffix";

set $rewrite_condition "$serve_markdown$has_no_extension";

if ($rewrite_condition = "11") {
set $proxy_path "${proxy_path}.md";
}

proxy_pass https://apify.github.io/apify-sdk-python$proxy_path;
}

### Repository path: "/api/client/js"

location = /api/client/js {
if ($serve_markdown) {
rewrite ^ /client-js-llms.txt last;
}
proxy_pass https://apify.github.io/apify-client-js/;
}
location /api/client/python {
location = /api/client/js/ {
if ($serve_markdown) {
rewrite ^ /client-js-llms.txt last;
}
proxy_pass https://apify.github.io/apify-client-js/;
}

location = /client-js-llms.txt {
proxy_hide_header Content-Type;
add_header Content-Type 'text/markdown; charset=utf-8' always;
proxy_pass https://apify.github.io/apify-client-js/llms.txt;
}

location ~ ^/api/client/js/(llms|llms-full)\.txt$ {
rewrite ^/api/client/js(/.*)$ $1 break;
proxy_hide_header Content-Type;
add_header Content-Type 'text/markdown; charset=utf-8' always;
proxy_pass https://apify.github.io/apify-client-js$uri;
}

location ~ ^/api/client/js/(.*)$ {
set $path_suffix $1;
set $proxy_path "/$path_suffix";

set $rewrite_condition "$serve_markdown$has_no_extension";

if ($rewrite_condition = "11") {
set $proxy_path "${proxy_path}.md";
}

proxy_pass https://apify.github.io/apify-client-js$proxy_path;
}

### Repository path: "/api/client/python"

location = /api/client/python {
if ($serve_markdown) {
rewrite ^ /client-python-llms.txt last;
}
proxy_pass https://apify.github.io/apify-client-python/;
}
location /sdk/js {
proxy_pass https://apify.github.io/apify-sdk-js/;
location = /api/client/python/ {
if ($serve_markdown) {
rewrite ^ /client-python-llms.txt last;
}
proxy_pass https://apify.github.io/apify-client-python/;
}
location /sdk/python {
proxy_pass https://apify.github.io/apify-sdk-python/;

location = /client-python-llms.txt {
proxy_hide_header Content-Type;
add_header Content-Type 'text/markdown; charset=utf-8' always;
proxy_pass https://apify.github.io/apify-client-python/llms.txt;
}

location ~ ^/api/client/python/(llms|llms-full)\.txt$ {
rewrite ^/api/client/python(/.*)$ $1 break;
proxy_hide_header Content-Type;
add_header Content-Type 'text/markdown; charset=utf-8' always;
proxy_pass https://apify.github.io/apify-client-python$uri;
}

location ~ ^/api/client/python/(.*)$ {
set $path_suffix $1;
set $proxy_path "/$path_suffix";

set $rewrite_condition "$serve_markdown$has_no_extension";

if ($rewrite_condition = "11") {
set $proxy_path "${proxy_path}.md";
}

proxy_pass https://apify.github.io/apify-client-python$proxy_path;
}
location /cli {

### Repository path: "/cli"

location = /cli {
if ($serve_markdown) {
rewrite ^ /cli-llms.txt last;
}
proxy_pass https://apify.github.io/apify-cli/;
}
location = /cli/ {
if ($serve_markdown) {
rewrite ^ /cli-llms.txt last;
}
proxy_pass https://apify.github.io/apify-cli/;
}

location = /cli-llms.txt {
proxy_hide_header Content-Type;
add_header Content-Type 'text/markdown; charset=utf-8' always;
proxy_pass https://apify.github.io/apify-cli/llms.txt;
}

location ~ ^/cli/(llms|llms-full)\.txt$ {
rewrite ^/cli(/.*)$ $1 break;
proxy_hide_header Content-Type;
add_header Content-Type 'text/markdown; charset=utf-8' always;
proxy_pass https://apify.github.io/apify-cli$uri;
}

location ~ ^/cli/(.*)$ {
set $path_suffix $1;
set $proxy_path "/$path_suffix";

set $rewrite_condition "$serve_markdown$has_no_extension";

if ($rewrite_condition = "11") {
set $proxy_path "${proxy_path}.md";
}

proxy_pass https://apify.github.io/apify-cli$proxy_path;
}

location = /health {
access_log off;
add_header 'Content-Type' 'application/json';
Expand Down Expand Up @@ -260,16 +449,16 @@ server {
rewrite ^/sdk/js/docs/typedefs(.*)$ /sdk/js/docs/2.3/typedefs$1 redirect;

# old integrated docs -> new docs in GH pages
rewrite ^/apify-client-js/?$ /api/client/js/ redirect;
rewrite ^/apify-client-js/latest/?$ /api/client/js/ redirect;
rewrite ^/apify-client-python/?$ /api/client/python/ redirect;

# add trailing slashes to the root of GH pages docs
rewrite ^/api/client/js$ /api/client/js/ redirect;
rewrite ^/api/client/python$ /api/client/python/ redirect;
rewrite ^/sdk/js$ /sdk/js/ redirect;
rewrite ^/sdk/python$ /sdk/python/ redirect;
rewrite ^/cli$ /cli/ redirect;
rewrite ^/apify-client-js/?$ /api/client/js redirect;
rewrite ^/apify-client-js/latest/?$ /api/client/js redirect;
rewrite ^/apify-client-python/?$ /api/client/python redirect;

# remove trailing slashes from the root of GH pages docs
rewrite ^/api/client/js/$ /api/client/js redirect;
rewrite ^/api/client/python/$ /api/client/python redirect;
rewrite ^/sdk/js/$ /sdk/js redirect;
rewrite ^/sdk/python/$ /sdk/python redirect;
rewrite ^/cli/$ /cli redirect;

# versions page redirects
rewrite ^/versions/?$ / permanent; # no docs-wide changelog, redirect to the root
Expand Down