refactor: Remove external libraries as they are hosted in our Conan Center Index fork (#5643)

This change:
* Removes the patched Conan recipes from the `external/` directory.
* Adds instructions for contributors how to obtain our patched recipes.
* Updates the Conan remote name and remote URL (the underlying package repository isn't changed).
* If the remote already exists, updates the URL instead of removing and re-adding.
  * This is not done for the libXRPL job as it still uses Conan 1. This job will be switched to Conan 2 soon.
* Removes duplicate Conan remote CI pipeline steps.
* Overwrites the existing global.conf on MacOS and Windows machines, as those do not run CI pipelines in isolation but all share the same Conan installation; appending the same config over and over bloats the file.

Author: bthomee

.github/actions/dependencies/action.yml

@@ -6,29 +6,17 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: export custom recipes
-      shell: bash
-      run: |
-        conan export --version 1.1.10 external/snappy
-        conan export --version 4.0.3 external/soci
-    - name: add Ripple Conan remote
+    - name: add Conan remote
       if: env.CONAN_URL != ''
       shell: bash
       run: |
-        if conan remote list | grep -q "ripple"; then
-            conan remote remove ripple
-            echo "Removed conan remote ripple"
+        if conan remote list | grep -q 'xrplf'; then
+          conan remote update --index 0 --url ${CONAN_URL} xrplf
+          echo "Updated Conan remote 'xrplf' to ${CONAN_URL}."
+        else
+          conan remote add --index 0 xrplf ${CONAN_URL}
+          echo "Added new conan remote 'xrplf' at ${CONAN_URL}."
         fi
-        conan remote add --index 0 ripple "${CONAN_URL}"
-        echo "Added conan remote ripple at ${CONAN_URL}"
-    - name: try to authenticate to Ripple Conan remote
-      if: env.CONAN_LOGIN_USERNAME_RIPPLE != '' && env.CONAN_PASSWORD_RIPPLE != ''
-      id: remote
-      shell: bash
-      run: |
-        echo "Authenticating to ripple remote..."
-        conan remote auth ripple --force
-        conan remote list-users
     - name: list missing binaries
       id: binaries
       shell: bash

.github/workflows/libxrpl.yml

@@ -1,8 +1,8 @@
 name: Check libXRPL compatibility with Clio
 env:
-  CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/dev
-  CONAN_LOGIN_USERNAME_RIPPLE: ${{ secrets.CONAN_USERNAME }}
-  CONAN_PASSWORD_RIPPLE: ${{ secrets.CONAN_TOKEN }}
+  CONAN_URL: https://conan.ripplex.io
+  CONAN_LOGIN_USERNAME_XRPLF: ${{ secrets.CONAN_USERNAME }}
+  CONAN_PASSWORD_XRPLF: ${{ secrets.CONAN_TOKEN }}
 on:
   pull_request:
     paths:
@@ -43,28 +43,28 @@ jobs:
         shell: bash
         run: |
           conan export . ${{ steps.channel.outputs.channel }}
-      - name: Add Ripple Conan remote
+      - name: Add Conan remote
         shell: bash
         run: |
           conan remote list
-          conan remote remove ripple || true
+          conan remote remove xrplf || true
           # Do not quote the URL. An empty string will be accepted (with a non-fatal warning), but a missing argument will not.
-          conan remote add ripple ${{ env.CONAN_URL }} --insert 0
+          conan remote add xrplf ${{ env.CONAN_URL }} --insert 0
       - name: Parse new version
         id: version
         shell: bash
         run: |
           echo version="$(cat src/libxrpl/protocol/BuildInfo.cpp | grep "versionString =" \
             | awk -F '"' '{print $2}')" | tee ${GITHUB_OUTPUT}
-      - name: Try to authenticate to Ripple Conan remote
+      - name: Try to authenticate to Conan remote
         id: remote
         shell: bash
         run: |
           # `conan user` implicitly uses the environment variables CONAN_LOGIN_USERNAME_<REMOTE> and CONAN_PASSWORD_<REMOTE>.
           # https://docs.conan.io/1/reference/commands/misc/user.html#using-environment-variables
           # https://docs.conan.io/1/reference/env_vars.html#conan-login-username-conan-login-username-remote-name
           # https://docs.conan.io/1/reference/env_vars.html#conan-password-conan-password-remote-name
-          echo outcome=$(conan user --remote ripple --password >&2 \
+          echo outcome=$(conan user --remote xrplf --password >&2 \
             && echo success || echo failure) | tee ${GITHUB_OUTPUT}
       - name: Upload new package
         id: upload

.github/workflows/macos.yml

@@ -18,9 +18,7 @@ concurrency:
 # This part of Conan configuration is specific to this workflow only; we do not want
 # to pollute conan/profiles directory with settings which might not work for others
 env:
-  CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/dev
-  CONAN_LOGIN_USERNAME_RIPPLE: ${{ secrets.CONAN_USERNAME }}
-  CONAN_PASSWORD_RIPPLE: ${{ secrets.CONAN_TOKEN }}
+  CONAN_URL: https://conan.ripplex.io
   CONAN_GLOBAL_CONF: |
     core.download:parallel={{os.cpu_count()}}
     core.upload:parallel={{os.cpu_count()}}
@@ -87,24 +85,9 @@ jobs:
           clang --version
       - name: configure Conan
         run : |
-          echo "${CONAN_GLOBAL_CONF}" >> $(conan config home)/global.conf
+          echo "${CONAN_GLOBAL_CONF}" > $(conan config home)/global.conf
           conan config install conan/profiles/ -tf $(conan config home)/profiles/
           conan profile show
-      - name: export custom recipes
-        shell: bash
-        run: |
-          conan export --version 1.1.10 external/snappy
-          conan export --version 4.0.3 external/soci
-      - name: add Ripple Conan remote
-        if: env.CONAN_URL != ''
-        shell: bash
-        run: |
-          if conan remote list | grep -q "ripple"; then
-              conan remote remove ripple
-              echo "Removed conan remote ripple"
-          fi
-          conan remote add --index 0 ripple "${CONAN_URL}"
-          echo "Added conan remote ripple at ${CONAN_URL}"
       - name: build dependencies
         uses: ./.github/actions/dependencies
         with:

.github/workflows/nix.yml

@@ -19,9 +19,7 @@ concurrency:
 # This part of Conan configuration is specific to this workflow only; we do not want
 # to pollute conan/profiles directory with settings which might not work for others
 env:
-  CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/dev
-  CONAN_LOGIN_USERNAME_RIPPLE: ${{ secrets.CONAN_USERNAME }}
-  CONAN_PASSWORD_RIPPLE: ${{ secrets.CONAN_TOKEN }}
+  CONAN_URL: https://conan.ripplex.io
   CONAN_GLOBAL_CONF: |
     core.download:parallel={{ os.cpu_count() }}
     core.upload:parallel={{ os.cpu_count() }}

.github/workflows/windows.yml

@@ -21,9 +21,7 @@ concurrency:
 # This part of Conan configuration is specific to this workflow only; we do not want
 # to pollute conan/profiles directory with settings which might not work for others
 env:
-  CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/dev
-  CONAN_LOGIN_USERNAME_RIPPLE: ${{ secrets.CONAN_USERNAME }}
-  CONAN_PASSWORD_RIPPLE: ${{ secrets.CONAN_TOKEN }}
+  CONAN_URL: https://conan.ripplex.io
   CONAN_GLOBAL_CONF: |
     core.download:parallel={{os.cpu_count()}}
     core.upload:parallel={{os.cpu_count()}}
@@ -82,24 +80,9 @@ jobs:
       - name: configure Conan
         shell: bash
         run: |
-          echo "${CONAN_GLOBAL_CONF}" >> $(conan config home)/global.conf
+          echo "${CONAN_GLOBAL_CONF}" > $(conan config home)/global.conf
           conan config install conan/profiles/ -tf $(conan config home)/profiles/
           conan profile show
-      - name: export custom recipes
-        shell: bash
-        run: |
-          conan export --version 1.1.10 external/snappy
-          conan export --version 4.0.3 external/soci
-      - name: add Ripple Conan remote
-        if: env.CONAN_URL != ''
-        shell: bash
-        run: |
-          if conan remote list | grep -q "ripple"; then
-              conan remote remove ripple
-              echo "Removed conan remote ripple"
-          fi
-          conan remote add --index 0 ripple "${CONAN_URL}"
-          echo "Added conan remote ripple at ${CONAN_URL}"
       - name: build dependencies
         uses: ./.github/actions/dependencies
         with:

BUILD.md

@@ -132,15 +132,28 @@ can do this by running:
 conan remote add --index 0 xrplf "https://conan.ripplex.io"
 ```
 
-Alternatively, you can use the `conan export` command to add the patched recipes
-that are also (for the time being) located in the `external/` directory to your
-local Conan cache:
+Alternatively, you can pull the patched recipes into the repository and use them
+locally:
 
 ```bash
-conan export --version 1.1.10 external/snappy
-conan export --version 4.0.3 external/soci
+cd external
+git init
+git remote add origin [email protected]:XRPLF/conan-center-index.git
+git sparse-checkout init
+git sparse-checkout set recipes/snappy
+git sparse-checkout add recipes/soci
+git fetch origin master
+git checkout master
+conan export --version 1.1.10 external/recipes/snappy
+conan export --version 4.0.3 external/recipes/soci
 ```
 
+In the case we switch to a newer version of a dependency that still requires a
+patch, it will be necessary for you to pull in the changes and re-export the
+updated dependencies with the newer version. However, if we switch to a newer
+version that no longer requires a patch, no action is required on your part, as
+the new recipe will be automatically pulled from the official Conan Center.
+
 ### Conan profile tweaks
 
 #### Missing compiler version
@@ -525,7 +538,7 @@ After any updates or changes to dependencies, you may need to do the following:
    conan remove '*'
    ```
 
-3. Re-run [conan export](#export-updated-recipes)
+3. Re-run [conan export](#patched-recipes) if needed.
 4. Re-run [conan install](#build-and-test).
 
 ### `protobuf/port_def.inc` file not found

external/README.md

@@ -1,14 +1,10 @@
 # External Conan recipes
 
-The subdirectories in this directory contain either copies or Conan recipes
-of external libraries used by rippled.
-The Conan recipes include patches we have not yet pushed upstream.
+The subdirectories in this directory contain copies of external libraries used
+by rippled.
 
 | Folder          | Upstream                                     | Description |
 |:----------------|:---------------------------------------------|:------------|
 | `antithesis-sdk`| [Project](https://github.com/antithesishq/antithesis-sdk-cpp/) | [Antithesis](https://antithesis.com/docs/using_antithesis/sdk/cpp/overview.html) SDK for C++ |
 | `ed25519-donna` | [Project](https://github.com/floodyberry/ed25519-donna) | [Ed25519](http://ed25519.cr.yp.to/) digital signatures |
-| `rocksdb`       | [Recipe](https://github.com/conan-io/conan-center-index/tree/master/recipes/rocksdb) | Fast key/value database. (Supports rotational disks better than NuDB.) |
 | `secp256k1`     | [Project](https://github.com/bitcoin-core/secp256k1)    | ECDSA digital signatures using the **secp256k1** curve |
-| `snappy`        | [Recipe](https://github.com/conan-io/conan-center-index/tree/master/recipes/snappy)  | "Snappy" lossless compression algorithm. |
-| `soci`          | [Recipe](https://github.com/conan-io/conan-center-index/tree/master/recipes/soci)    | Abstraction layer for database access. |