docs

Author: shreemaan-abhishek

docs/en/latest/plugins/ai-proxy-multi.md

@@ -193,3 +193,58 @@ curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \
 ```
 
 In the above configuration `priority` for the deepseek provider is set to `0`. Which means if `openai` provider is unavailable then `ai-proxy-multi` plugin will retry sending request to `deepseek` in the second attempt.
+
+### Send request to an OpenAI compatible LLM
+
+Create a route with the `ai-proxy-multi` plugin with `provider.name` set to `openai-compatible` and the endpoint of the model set to `provider.override.endpoint` like so:
+
+```shell
+curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \
+  -H "X-API-KEY: ${ADMIN_API_KEY}" \
+  -d '{
+    "id": "ai-proxy-multi-route",
+    "uri": "/anything",
+    "methods": ["POST"],
+    "plugins": {
+      "ai-proxy-multi": {
+        "providers": [
+          {
+            "name": "openai-compatible",
+            "model": "qwen-plus",
+            "weight": 1,
+            "priority": 1,
+            "auth": {
+              "header": {
+                "Authorization": "Bearer '"$OPENAI_API_KEY"'"
+              }
+            },
+            "override": {
+              "endpoint": "https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions"
+            }
+          },
+          {
+            "name": "deepseek",
+            "model": "deepseek-chat",
+            "weight": 1,
+            "auth": {
+              "header": {
+                "Authorization": "Bearer '"$DEEPSEEK_API_KEY"'"
+              }
+            },
+            "options": {
+                "max_tokens": 512,
+                "temperature": 1.0
+            }
+          }
+        ],
+        "passthrough": false
+      }
+    },
+    "upstream": {
+      "type": "roundrobin",
+      "nodes": {
+        "httpbin.org": 1
+      }
+    }
+  }'
+```

docs/en/latest/plugins/ai-proxy.md

@@ -142,3 +142,39 @@ You will receive a response like this:
   "usage": { "completion_tokens": 15, "prompt_tokens": 23, "total_tokens": 38 }
 }
 ```
+
+### Send request to an OpenAI compatible LLM
+
+Create a route with the `ai-proxy` plugin with `provider` set to `openai-compatible` and the endpoint of the model set to `override.endpoint` like so:
+
+```shell
+curl "http://127.0.0.1:9180/apisix/admin/routes/1" -X PUT \
+  -H "X-API-KEY: ${ADMIN_API_KEY}" \
+  -d '{
+    "uri": "/anything",
+    "plugins": {
+      "ai-proxy": {
+        "auth": {
+          "header": {
+            "Authorization": "Bearer <some-token>"
+          }
+        },
+        "model": {
+          "provider": "openai-compatible",
+          "name": "qwen-plus"
+        },
+        "override": {
+          "endpoint": "https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions"
+        }
+      }
+    },
+    "upstream": {
+      "type": "roundrobin",
+      "nodes": {
+        "somerandom.com:443": 1
+      },
+      "scheme": "https",
+      "pass_host": "node"
+    }
+  }'
+```