Merge pull request #109 from jagerman/docs-day-2022-04

API docs generation using docsify

Author: jagerman

docs/make-api-docs.py renamed to docs/generate-api-docs.py

@@ -6,6 +6,7 @@
 import os
 import argparse
 import sys
+import shutil
 
 parser = argparse.ArgumentParser()
 parser.add_argument(
@@ -18,11 +19,18 @@
     "top-level headings start with a single `#`, sub-headings start with `##`, etc.  For example, "
     "3 would start with `###`, sub-headings would be `####`, etc.",
 )
+parser.add_argument(
+    "-m",
+    "--multi-file",
+    action="store_true",
+    help="Write to one markdown file per section, rather than one single large file",
+)
 parser.add_argument(
     "-o",
     "--output",
     type=str,
-    help="Specify output file; specifing a - or omitting outputs to stdout",
+    help="Specify output file (or directory, when using -m); specifing a - or omitting outputs to "
+    "stdout, but is not accepted in multi-file (-m) mode.",
 )
 parser.add_argument(
     "-W",
@@ -34,7 +42,11 @@
 args = parser.parse_args()
 
 out_file = False
-if args.output and args.output != '-':
+if args.multi_file and (not args.output or args.output == '-'):
+    print("-o must specify a directory when using -m")
+    sys.exit(1)
+
+if not args.multi_file and args.output and args.output != '-':
     if not args.overwrite and os.path.exists(args.output):
         print(f"{args.output} already exists; remove it first or use --overwrite/-W to overwrite")
         sys.exit(1)
@@ -71,20 +83,23 @@ def read_snippet(markdown, depth=args.markdown_level):
     return None
 
 
-section_list, sections = [], {}
+section_list, section_names, section_snips, sections = [], [], [], {}
 for name, bp in app.blueprints.items():
     s = []
     section_list.append(s)
+    section_names.append(name)
     sections[name] = s
     snip = read_snippet(f'{name}.md')
     if snip:
         s.append(snip)
     else:
         s.append(f"{h1} {name.title()}\n\n")
         app.logger.warning(f"{name}.md not found: adding stub '{name.title()}' section")
+    section_snips.append(snip)
 
 # Last section is for anything with a not found category:
 section_list.append([])
+section_names.append('uncategorized')
 
 
 # Sort endpoints within a section by the number of URL parts, first, then alphabetically because we
@@ -95,6 +110,8 @@ def endpoint_sort_key(rule):
 
 for rule in sorted(app.url_map.iter_rules(), key=endpoint_sort_key):
     ep = rule.endpoint
+    if ep == 'static':
+        continue
     methods = [m for m in rule.methods if m not in ('OPTIONS', 'HEAD')]
     if not methods:
         app.logger.warning(f"Endpoint {ep} has no useful method, skipping!")
@@ -109,6 +126,11 @@ def endpoint_sort_key(rule):
     handler = app.view_functions[ep]
 
     doc = handler.__doc__
+    if ep.startswith('legacy'):
+        # We deliberately omit legacy endpoint documentation
+        if doc is not None:
+            app.logger.warning(f"Legacy endpoint {ep} has docstring but it will be omitted")
+        continue
     if doc is None:
         app.logger.warning(f"Endpoint {ep} has no docstring!")
         doc = '*(undocumented)*'
@@ -174,7 +196,9 @@ def endpoint_sort_key(rule):
                 argdoc = argdoc.replace('\n', '\n  ')
 
                 if ':`' in argdoc:
-                    app.logger.warning(f"{arg} still contains some rst crap we need to handle")
+                    app.logger.warning(
+                        f"{method} {url} ({arg}) still contains some rst crap we need to handle"
+                    )
 
                 s.append(f" — {argdoc}\n\n")
             else:
@@ -189,30 +213,90 @@ def endpoint_sort_key(rule):
 
     more = read_snippet(f'{ep}.md', depth=3)
     if more:
+        s.append("\n\n")
         s.append(more)
 
     s.append("\n\n\n")
 
 
-out = open(args.output, 'w') if out_file else sys.stdout
+out = open(args.output, 'w') if out_file else sys.stdout if not args.multi_file else None
 
 if section_list[-1]:
     # We have some uncategorized entries, so load the .md for it
     other = read_snippet('uncategorized.md')
-    if other:
-        section_list[-1].insert(0, other)
-    else:
+    if not other:
         app.logger.warning(
             "Found uncategorized sections, but uncategorized.md not found; inserting stub"
         )
-        section_list[-1].insert(0, "# Uncategorized Endpoints\n\n")
+        other = "# Uncategorized Endpoints\n\n"
+
+    section_list[-1].insert(0, other)
+    section_snips.append(other)
+
+else:
+    section_list.pop()
+    section_names.pop()
+
+if args.multi_file:
+    if not os.path.exists(args.output):
+        os.makedirs(args.output)
+    if not args.overwrite and os.path.exists(args.output + '/index.md'):
+        print(f"{args.output}/index.md already exists; remove it first or use --overwrite/-W")
+        sys.exit(1)
+
+    api_readme_f = open(args.output + '/index.md', 'w')
+
+section_order = range(0, len(section_list))
+if args.multi_file:
+    # In multi-file mode we take the order for the index file section from the order it is listed in
+    # sidebar.md:
+    sidebar = read_snippet('sidebar.md')
+    shutil.copy(snippets + '/sidebar.md', args.output + '/sidebar.md')
+
+    def pos(i):
+        x = sidebar.find(section_names[i])
+        return x if x >= 0 else len(sidebar)
+
+    section_order = sorted(section_order, key=pos)
+
+for i in section_order:
+    if args.multi_file:
+        filename = args.output + '/' + section_names[i] + '.md'
+        if not args.overwrite and os.path.exists(filename):
+            print(f"{filename} already exists; remove it first or use --overwrite/-W to overwrite")
+            sys.exit(1)
+        if out is not None:
+            out.close()
+        out = open(filename, 'w')
+
+        if section_names[i] + '.md' not in sidebar:
+            app.logger.warning(
+                f"{section_names[i]}.md not found in snippets/sidebar.md: "
+                "section will be missing from the sidebar!"
+            )
+
+        snip = section_snips[i]
+        if snip.startswith(f'{h1} '):
+            preamble = snip.find(f'{h2}')
+            if preamble == -1:
+                preamble = snip
+            else:
+                preamble = snip[:preamble]
+            print(
+                re.sub(fr'^{h1} (.*)', fr'{h1} [\1]({section_names[i]}.md)', preamble),
+                file=api_readme_f,
+            )
+        else:
+            app.logger.warning(
+                f"{section_names[i]} section didn't start with expected '# Title', "
+                f"cannot embed section link in {args.output}/index.md"
+            )
 
-for s in section_list:
-    for x in s:
+    for x in section_list[i]:
         print(x, end='', file=out)
     print("\n\n", file=out)
 
-if out_file:
+if out is not None and out != sys.stdout:
     out.close()
 
 app.logger.info("API doc created successfully!")

docs/make-docs.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+set -e
+
+if [ "$(basename $(pwd))" != "docs" ] || ! [ -e "make-docs.sh" ]; then
+    echo "Error: you must run this from the docs directory" >&2
+    exit 1
+fi
+
+rm -rf api
+
+docsify init --local api
+
+rm -f api/README.md
+
+if [ -n "$NPM_PACKAGES" ]; then
+    npm_dir="$NPM_PACKAGES/lib/node_modules"
+elif [ -n "$NODE_PATH" ]; then
+    npm_dir="$NODE_PATH"
+elif [ -d "$HOME/node_modules" ]; then
+    npm_dir="$HOME/node_modules"
+elif [ -d "/usr/local/lib/node_modules" ]; then
+    npm_dir="/usr/local/lib/node_modules"
+else
+    echo "Can't determine your node_modules path; set NPM_PACKAGES or NODE_PATH appropriately" >&2
+    exit 1
+fi
+
+cp $npm_dir/docsify/node_modules/prismjs/components/prism-{json,python,http}.min.js api/vendor
+cp $npm_dir/docsify-katex/dist/docsify-katex.js api/vendor
+cp $npm_dir/docsify-katex/node_modules/katex/dist/katex.min.css api/vendor
+
+PYTHONPATH=.. ./generate-api-docs.py -m -o api
+
+perl -ni.bak -e '
+BEGIN { $first = 0; }
+if (m{^\s*<script>\s*$} .. m{^\s*</script>\s*$}) {
+    if (not $first) {
+        $first = false;
+        print qq{
+  <script>
+    window.\$docsify = {
+      name: "Session PySOGS API",
+      repo: "https://github.com/oxen-io/session-pysogs",
+      loadSidebar: "sidebar.md",
+      subMaxLevel: 2,
+      homepage: "index.md",
+    }
+  </script>\n};
+    }
+} else {
+    s{<title>.*</title>}{<title>Session PySOGS API</title>};
+    s{(name="description" content=)"[^"]*"}{$1"Session PySOGS API documentation"};
+    if (m{^\s*</body>}) {
+        print qq{
+  <script src="vendor/prism-json.min.js"></script>
+  <script src="vendor/prism-python.min.js"></script>
+  <script src="vendor/prism-http.min.js"></script>
+  <script src="vendor/docsify-katex.js"></script>
+  <link rel="stylesheet" href="vendor/katex.min.css" />
+};
+    }
+    print;
+}' api/index.html
+

docs/mkdocs.yml

@@ -1,41 +1,49 @@
-# DMs
+# Direct Messages
 
-Direct messages between blinded users
+These endpoints are used for sending and retrieving direct messages between blinded users.  Such
+messages are only used for introductions; once both parties know each other's Session ID messages
+should be done via regular Session one-to-one messaging.
 
 ## Encryption details
 
 SOGS itself does not have the ability to decrypt the message contents and thus cannot enforce
 any particular content; the following, however, is strongly recommended for Session client
 interoperability:
 
-Alice has master Session Ed25519 keypair `a`, `A` and blinded keypair `ka`, `kA`.
+Alice has master Session Ed25519 keypair $a$, $A$ and blinded keypair $ka$, $kA$.
 
-Bob has master Session Ed25519 keypair `b`, `B` and blinded keypair `kb`, `kB`.
+Bob has master Session Ed25519 keypair $b$, $B$ and blinded keypair $kb$, $kB$.
 
-Alice wants to send a message to Bob, knowing only `kB` (i.e. the blinded Session ID after
+Alice wants to send a message $M$ to Bob, knowing only $kB$ (i.e. the blinded Session ID after
 stripping the `0x15` prefix).
 
 Alice constructs a message using Session protobuf encoding, then concatenates her *unblinded*
-pubkey, `A`, to this message:
+pubkey, $A$, to this message:
 
-    MSG || A
+$$
+    M \parallel A
+$$
 
 Alice then constructs an encryption key:
 
-    E = H(a * kB || kA || kB)
+$$
+    E = H(a * kB \parallel kA \parallel kB)
+$$
 
-where `H(.)` is 32-byte BLAKE2b, and the `*` denotes unclamped Ed25519 scalar*point multiplication
+where $H(\cdot)$ is 32-byte BLAKE2b, and the $*$ denotes unclamped Ed25519 scalar*point multiplication
 (e.g. libsodium's `crypto_scalarmult_ed25519_noclamp`).
 
-The `MSG || A` plaintext value is then encrypted using XChaCha20-Poly1305 (e.g. using
+The $M \parallel A$ plaintext value is then encrypted using XChaCha20-Poly1305 (e.g. using
 libsodium's `crypto_aead_xchacha20poly1305_ietf_encrypt` function), using a secure-random
-24-byte nonce, no additional data, and encryption key `E`.
+24-byte nonce, no additional data, and encryption key $E$.
 
 The final data message is then constructed as:
 
-    0x00 || CIPHERTEXT || NONCE
+$$
+    \texttt{0x00} \parallel \textit{Ciphertext} \parallel \textit{Nonce}
+$$
 
-where 0x00 is a version byte (allowing for future alternative encryption formats) and the rest
+where $0x00$ is a version byte (allowing for future alternative encryption formats) and the rest
 are bytes.
 
 Finally this is base64-encoded when send to/retrieved from the SOGS.
@@ -48,23 +56,25 @@ Decryption proceeds by reversing the steps above:
 
 2. Grab the version byte from the front and the 24-byte nonce from the back of the value.
 
-    a) if the version byte is not `0x00` abort because this message is from someone using a
+    a) if the version byte is not $0x00$ abort because this message is from someone using a
     different encryption protocol.
 
 3. Construct the encryption key by calculating:
 
+    $$
         E = H(b * kA || kA || kB)
+    $$
 
-    where `kA` is the sender's de-prefixed blinded Session ID, `b` is the user's master Ed25519 key,
-    and `kB` is the user's blinded Ed25519 for this SOGS server.
+    where $kA$ is the sender's de-prefixed blinded Session ID, $b$ is the user's master Ed25519 key,
+    and $kB$ is the user's blinded Ed25519 for this SOGS server.
 
 4. Decrypt the remaining ciphertext using the nonce.
 
-5. Unpack the plaintext value into MSG and the sender's `A` value (i.e. the last 32 bytes).
+5. Unpack the plaintext value into MSG and the sender's $A$ value (i.e. the last 32 bytes).
 
-6. Derive the sender's actual Session ID by converting `A` from an ed25519 pubkey to an
+6. Derive the sender's actual Session ID by converting $A$ from an ed25519 pubkey to an
    curve25519 pubkey and prepending 0x05.  (E.g. using libsodium's
-   `crypto_sign_ed25519_pk_to_curve25519` on `A`, then adding the `05` prefix to the front).
+   `crypto_sign_ed25519_pk_to_curve25519` on $A$, then adding the `05` prefix to the front).
 
 This then leaves the receiving client with the true Session ID of the sender, and the message
 body (encoded according to typical Session message protobuf encoding).

docs/snippets/dm.md

@@ -0,0 +1,25 @@
+# Example retrieving capabilities:
+
+```http
+GET /capabilities HTTP/1.1
+Host: example.com
+```
+
+```json
+{
+  "capabilities": ["sogs", "batch"]
+}
+```
+
+# Example with capability check
+
+```http
+GET /capabilities?required=magic,batch HTTP/1.1
+```
+
+```json
+{
+  "capabilities": ["sogs", "batch"],
+  "missing": ["magic"]
+}
+```

docs/snippets/general.get_caps.md

@@ -0,0 +1,8 @@
+- [General](general.md)
+- [Rooms](rooms.md)
+- [Messages](messages.md)
+- [User management](users.md)
+- [Direct messages](dm.md)
+- [Onion requests](onion_request.md)
+- [Web viewer](views.md)
+- [Deprecated legacy SOGS](legacy.md)