DOC-689 | Fix toolchain, improve cURL example rendering (#538)

* Check for server ready using IPv4 localhost

* Re-add heredoc for curl requests with body, fix rendering of command

* headers object always defined at this point

* Remove highlight.js

* Fix README link to datasets.json

* Replace batch request examples with manual code

logCurlRequestPlain is not compatible with new toolchain

* Don't unnecessarily JSON.stringify payload in /_api/collection/{coll}/responsibleShard example

* Try non-raw logCurlRequest for import examples

* Using an inline regex seems to break the toolchain?

The second openRepoStream message is missing and arangoproxy not responding to requests if you do?!

* Remove Docker Compose version field

* Typo

* Trying to escape header values seems to break the toolchain

* Make batch request examples generated again

* Apply to 3.10, 3.11, 3.13

* Skip compilation for ArangoDB for arangodb/enterprise too

* Strip off hotfix suffix from image tag to determine upstream branch

* Also handle enterprise image when pulling

* Guard against sporadic failure in some async job examples

Fetching the result can return HTTP 204 which means the job is still queued

* [skip ci] Automatic commit of generated files from CircleCI

* Add comment about alternative solution

---------

Co-authored-by: CircleCI Job <[email protected]>

Author: Simran-B

.circleci/base_config.yml

@@ -20,7 +20,8 @@ commands:
 
             if [[ "<< parameters.branch >>" == *"arangodb/enterprise"* ]]; then
                 echo "[SETUP] An official ArangoDB Enterprise image has been chosen"
-                branch_name=$(echo << parameters.branch >> | cut -d: -f2 | cut -d- -f1)
+                echo "[SETUP] Using image tag as branch name (with hotfix suffixes removed)"
+                branch_name=$(echo << parameters.branch >> | cut -d: -f2 | cut -d- -f1 | sed -E 's/^([0-9]+\.[0-9]+\.[0-9]+).+/\1/')
             fi
 
             echo "[SETUP] Git clone $branch_name"

.circleci/generate_config.py

@@ -112,7 +112,7 @@ def workflow_generate(config):
             }
         }
 
-        if not "enterprise-preview" in branch:
+        if not "/enterprise-preview:" in branch and not "/enterprise:" in branch:
 
             openssl = findOpensslVersion(branch)
             compileJob["compile-linux"]["openssl"] = openssl
@@ -409,7 +409,7 @@ def workflow_release_launch_command(config):
 def pullImageCmd(branch, version):
     pullImage = f"docker pull {branch}"
 
-    if not "enterprise-preview" in branch:
+    if not "/enterprise-preview:" in branch and not "/enterprise:" in branch:
         pullImage = f"BRANCH={branch}\n\
 version={version}\n"
         pullImage += "\

README.md

@@ -1282,7 +1282,7 @@ loaded before the query is run:
 dataset: name_of_dataset
 ```
 
-See [datasets.json](toolchain/arangoproxy/internal/utils/)
+See [datasets.json](toolchain/arangoproxy/internal/utils/datasets.json)
 for the available datasets.
 
 To get the query explain output including the execution plan instead of the

site/content/3.10/develop/http-api/batch-requests.md

@@ -305,18 +305,16 @@ description: |-
   Sending a batch request with five batch parts:
 
   - GET /_api/version
-
   - DELETE /_api/collection/products
-
   - POST /_api/collection/products
-
   - GET /_api/collection/products/figures
-
   - DELETE /_api/collection/products
 
   The boundary (`SomeBoundaryValue`) is passed to the server in the HTTP
   `Content-Type` HTTP header.
-  *Please note the reply is not displayed all accurate.*
+
+  The server response is formatted for readability. The `↩` character denotes
+  the original line breaks.
 name: RestBatchMultipartHeader
 ---
 var parts = [
@@ -342,13 +340,12 @@ var parts = [
   "DELETE /_api/collection/products HTTP/1.1\r\n"
 ];
 var boundary = "SomeBoundaryValue";
-var headers = { "Content-Type" : "multipart/form-data; boundary=" +
-boundary };
+var headers = { "Content-Type" : "multipart/form-data; boundary=" + boundary };
 var body = "--" + boundary + "\r\n" +
            parts.join("\r\n" + "--" + boundary + "\r\n") +
            "--" + boundary + "--\r\n";
 
-var response = logCurlRequestPlain('POST', '/_api/batch', body, headers);
+var response = logCurlRequest('POST', '/_api/batch', body, headers);
 
 assert(response.code === 200);
 
@@ -358,8 +355,11 @@ logPlainResponse(response);
 ```curl
 ---
 description: |-
-  Sending a batch request, setting the boundary implicitly (the server will
-  in this case try to find the boundary at the beginning of the request body).
+  Sending a batch request, setting the boundary implicitly. The server tries
+  to find the boundary at the beginning of the request body in this case.
+
+  The server response is formatted for readability. The `↩` character denotes
+  the original line breaks.
 name: RestBatchImplicitBoundary
 ---
 var parts = [
@@ -373,7 +373,7 @@ var body = "--" + boundary + "\r\n" +
            parts.join("\r\n" + "--" + boundary + "\r\n") +
            "--" + boundary + "--\r\n";
 
-var response = logCurlRequestPlain('POST', '/_api/batch', body);
+var response = logCurlRequest('POST', '/_api/batch', body);
 
 assert(response.code === 200);
 assert(response.headers['x-arango-errors'] == 2);

site/content/3.10/develop/http-api/collections.md

@@ -686,8 +686,8 @@ var cn = "testCollection";
 db._drop(cn);
 db._create(cn, { numberOfShards: 3, shardKeys: ["_key"] });
 
-var body = JSON.stringify({ _key: "testkey", value: 23 });
-var response = logCurlRequestRaw('PUT', "/_api/collection/" + cn + "/responsibleShard", body);
+var body = { _key: "testkey", value: 23 };
+var response = logCurlRequest('PUT', "/_api/collection/" + cn + "/responsibleShard", body);
 
 assert(response.code === 200);
 assert(response.parsedBody.hasOwnProperty("shardId"));

site/content/3.10/develop/http-api/import.md

@@ -255,7 +255,7 @@ var body = [
   { name: { detailed: "detailed name", short: "short name" } }
 ];
 
-var response = logCurlRequestRaw('POST', "/_api/import?collection=" + cn + "&type=list", body);
+var response = logCurlRequest('POST', "/_api/import?collection=" + cn + "&type=list", body);
 
 assert(response.code === 201);
 assert(response.parsedBody.created === 3);
@@ -283,8 +283,7 @@ var body = '{ "_key": "abc", "value1": 25, "value2": "test",' +
            '{ "_key": "foo", "name": "baz" }\n\n' +
            '{ "name": {' +
            ' "detailed": "detailed name", "short": "short name" } }\n';
-var response = logCurlRequestRaw('POST', "/_api/import?collection=" + cn
-+ "&type=documents", body);
+var response = logCurlRequest('POST', "/_api/import?collection=" + cn + "&type=documents", body);
 
 assert(response.code === 201);
 assert(response.parsedBody.created === 3);

site/content/3.10/develop/http-api/jobs.md

@@ -94,7 +94,17 @@ assert(response.code === 202);
 logRawResponse(response);
 
 var queryId = response.headers['x-arango-async-id'];
-url = '/_api/job/' + queryId
+url = '/_api/job/' + queryId;
+
+for (let i = 1; i <= 10; i++) {
+  var status = arango.GET_RAW(url);
+  if (!status.headers['x-arango-async-id'] && status.code == 204) {
+    internal.sleep(0.1 * i * i);
+  } else {
+    break;
+  }
+}
+
 var response = logCurlRequest('PUT', url, "");
 assert(response.code === 200);
 logJsonResponse(response);
@@ -114,7 +124,17 @@ assert(response.code === 202);
 logRawResponse(response);
 
 var queryId = response.headers['x-arango-async-id'];
-url = '/_api/job/' + queryId
+url = '/_api/job/' + queryId;
+
+for (let i = 1; i <= 10; i++) {
+  var status = arango.GET_RAW(url);
+  if (!status.headers['x-arango-async-id'] && status.code == 204) {
+    internal.sleep(0.1 * i * i);
+  } else {
+    break;
+  }
+}
+
 var response = logCurlRequest('PUT', url, "");
 assert(response.code === 400);
 logJsonResponse(response);

site/content/3.11/develop/http-api/batch-requests.md

@@ -305,18 +305,16 @@ description: |-
   Sending a batch request with five batch parts:
 
   - GET /_api/version
-
   - DELETE /_api/collection/products
-
   - POST /_api/collection/products
-
   - GET /_api/collection/products/figures
-
   - DELETE /_api/collection/products
 
   The boundary (`SomeBoundaryValue`) is passed to the server in the HTTP
   `Content-Type` HTTP header.
-  *Please note the reply is not displayed all accurate.*
+
+  The server response is formatted for readability. The `↩` character denotes
+  the original line breaks.
 name: RestBatchMultipartHeader
 ---
 var parts = [
@@ -342,13 +340,12 @@ var parts = [
   "DELETE /_api/collection/products HTTP/1.1\r\n"
 ];
 var boundary = "SomeBoundaryValue";
-var headers = { "Content-Type" : "multipart/form-data; boundary=" +
-boundary };
+var headers = { "Content-Type" : "multipart/form-data; boundary=" + boundary };
 var body = "--" + boundary + "\r\n" +
            parts.join("\r\n" + "--" + boundary + "\r\n") +
            "--" + boundary + "--\r\n";
 
-var response = logCurlRequestPlain('POST', '/_api/batch', body, headers);
+var response = logCurlRequest('POST', '/_api/batch', body, headers);
 
 assert(response.code === 200);
 
@@ -358,8 +355,11 @@ logPlainResponse(response);
 ```curl
 ---
 description: |-
-  Sending a batch request, setting the boundary implicitly (the server will
-  in this case try to find the boundary at the beginning of the request body).
+  Sending a batch request, setting the boundary implicitly. The server tries
+  to find the boundary at the beginning of the request body in this case.
+
+  The server response is formatted for readability. The `↩` character denotes
+  the original line breaks.
 name: RestBatchImplicitBoundary
 ---
 var parts = [
@@ -373,7 +373,7 @@ var body = "--" + boundary + "\r\n" +
            parts.join("\r\n" + "--" + boundary + "\r\n") +
            "--" + boundary + "--\r\n";
 
-var response = logCurlRequestPlain('POST', '/_api/batch', body);
+var response = logCurlRequest('POST', '/_api/batch', body);
 
 assert(response.code === 200);
 assert(response.headers['x-arango-errors'] == 2);

site/content/3.11/develop/http-api/collections.md

@@ -686,8 +686,8 @@ var cn = "testCollection";
 db._drop(cn);
 db._create(cn, { numberOfShards: 3, shardKeys: ["_key"] });
 
-var body = JSON.stringify({ _key: "testkey", value: 23 });
-var response = logCurlRequestRaw('PUT', "/_api/collection/" + cn + "/responsibleShard", body);
+var body = { _key: "testkey", value: 23 };
+var response = logCurlRequest('PUT', "/_api/collection/" + cn + "/responsibleShard", body);
 
 assert(response.code === 200);
 assert(response.parsedBody.hasOwnProperty("shardId"));

site/content/3.11/develop/http-api/import.md

@@ -255,7 +255,7 @@ var body = [
   { name: { detailed: "detailed name", short: "short name" } }
 ];
 
-var response = logCurlRequestRaw('POST', "/_api/import?collection=" + cn + "&type=list", body);
+var response = logCurlRequest('POST', "/_api/import?collection=" + cn + "&type=list", body);
 
 assert(response.code === 201);
 assert(response.parsedBody.created === 3);
@@ -283,8 +283,7 @@ var body = '{ "_key": "abc", "value1": 25, "value2": "test",' +
            '{ "_key": "foo", "name": "baz" }\n\n' +
            '{ "name": {' +
            ' "detailed": "detailed name", "short": "short name" } }\n';
-var response = logCurlRequestRaw('POST', "/_api/import?collection=" + cn
-+ "&type=documents", body);
+var response = logCurlRequest('POST', "/_api/import?collection=" + cn + "&type=documents", body);
 
 assert(response.code === 201);
 assert(response.parsedBody.created === 3);