diff --git a/.github/workflows/internal-build.yml b/.github/workflows/internal-build.yml index d6fd2a80f..b76c9a67b 100644 --- a/.github/workflows/internal-build.yml +++ b/.github/workflows/internal-build.yml @@ -284,6 +284,12 @@ jobs: pattern: image-lab-${{ steps.ids.outputs.lab }}-${{ matrix.arch }}.* merge-multiple: true path: /tmp/images + - name: Download Image Galexie + uses: actions/download-artifact@v4 + with: + pattern: image-galexie-${{ steps.ids.outputs.galexie }}-${{ matrix.arch }}.* + merge-multiple: true + path: /tmp/images - name: Load Image into Docker run: | ls -lah /tmp/images/ diff --git a/.github/workflows/internal-test.yml b/.github/workflows/internal-test.yml index 90f61ee2f..331eb07fa 100644 --- a/.github/workflows/internal-test.yml +++ b/.github/workflows/internal-test.yml @@ -66,7 +66,7 @@ jobs: image: ${{ fromJSON(inputs.images) }} arch: ${{ fromJSON(inputs.archs) }} network: ["local"] - enable: ["core","rpc","core,rpc,horizon"] + enable: ["core","rpc","core,rpc,horizon","galexie"] options: [""] include: ${{ fromJSON(needs.setup.outputs.additional-tests) }} fail-fast: false @@ -172,6 +172,13 @@ jobs: docker logs stellar -f & echo "supervisorctl tail -f stellar-rpc" | docker exec -i stellar sh & go run tests/test_stellar_rpc_healthy.go + - name: Run galexie test + if: ${{ contains(matrix.enable, 'galexie') }} + timeout-minutes: ${{ fromJSON(steps.timeout.outputs.minutes) }} + run: | + docker logs stellar -f & + echo "supervisorctl tail -f galexie" | docker exec -i stellar sh & + go run tests/test_galexie.go - name: Prepare Test Logs if: always() run: | diff --git a/Dockerfile b/Dockerfile index 4587eec0e..1799e19a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,7 @@ ARG HORIZON_IMAGE=stellar-horizon-stage ARG FRIENDBOT_IMAGE=stellar-friendbot-stage ARG RPC_IMAGE=stellar-rpc-stage ARG LAB_IMAGE=stellar-lab-stage +ARG GALEXIE_IMAGE=stellar-galexie-stage # xdr @@ -187,6 +188,24 @@ COPY --from=stellar-lab-builder /lab/public /lab/public COPY --from=stellar-lab-builder /lab/build/static /lab/public/_next/static COPY --from=stellar-lab-builder /usr/local/bin/node /node +# galexie + +FROM golang:1.24-trixie AS stellar-galexie-builder + +ARG GALEXIE_REPO +ARG GALEXIE_REF + +WORKDIR /src +RUN git clone https://github.com/${GALEXIE_REPO} /src +RUN git fetch origin ${GALEXIE_REF} +RUN git checkout ${GALEXIE_REF} +ENV CGO_ENABLED=0 +RUN go build -o /galexie . + +FROM scratch AS stellar-galexie-stage + +COPY --from=stellar-galexie-builder /galexie /galexie + # quickstart FROM $XDR_IMAGE AS xdr @@ -195,6 +214,7 @@ FROM $HORIZON_IMAGE AS horizon FROM $FRIENDBOT_IMAGE AS friendbot FROM $RPC_IMAGE AS rpc FROM $LAB_IMAGE AS lab +FROM $GALEXIE_IMAGE AS galexie FROM ubuntu:24.04 AS quickstart @@ -221,6 +241,7 @@ COPY --from=friendbot /friendbot /usr/local/bin/friendbot COPY --from=rpc /stellar-rpc /usr/bin/stellar-rpc COPY --from=lab /lab /opt/stellar/lab COPY --from=lab /node /usr/bin/ +COPY --from=galexie /galexie /usr/bin/galexie RUN adduser --system --group --quiet --home /var/lib/stellar --disabled-password --shell /bin/bash stellar; diff --git a/README.md b/README.md index 7bba2e09e..19c0bdfb1 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ The image runs the following software: - [stellar-horizon](https://github.com/stellar/stellar-horizon) - API server - [stellar-friendbot](https://github.com/stellar/friendbot) - Faucet - [stellar-lab](https://github.com/stellar/laboratory) - Web UI +- [galexie](https://github.com/stellar/galexie) - Ledger meta exporter - [postgresql](https://www.postgresql.org) 12 is used for storing horizon data. - [supervisord](http://supervisord.org) is used from managing the processes of the above services. @@ -57,6 +58,7 @@ HTTP APIs and Tools are available at the following port and paths: - RPC: `http://localhost:8000/rpc` - Lab: `http://localhost:8000/lab` - Friendbot: `http://localhost:8000/friendbot` +- Ledger Meta: `http://localhost:8000/ledger-meta` (available with `--local` and `--enable galexie`) - History Archive: `http://localhost:8000/archive` (available with `--local` only) ## Tags @@ -257,6 +259,27 @@ $ curl http://localhost:8000/friendbot?addr=G... _Note: In local mode a local friendbot is running. In testnet and futurenet modes requests to the local `:8000/friendbot` endpoint will be proxied to the friendbot deployments for the respective network._ +### Galexie (Ledger Meta Exporter) + +Galexie is a ledger meta exporter that captures ledger close meta, which contains transaction meta, from the network and stores it locally. The exported ledger meta follows the [SEP-54](https://github.com/stellar/stellar-protocol/blob/master/ecosystem/sep-0054.md) format and is served via a file server. + +The ledger meta is only available in `--local` network mode. To enable it use the `--enable` option to specify all the services you wish to run and include `galexie`. + +``` +docker run -it -p 8000:8000 stellar/quickstart --local --enable core,rpc,galexie +``` + + +The ledger meta is available at: + +``` +http://localhost:8000/ledger-meta +``` + +The ledger meta store includes: +- `.config.json` - Configuration file describing the data format +- Partition directories containing compressed XDR files compressed with zstd with one ledger per file (`.xdr.zst`) + ### Using in GitHub Actions The quickstart image can be run in GitHub Actions workflows using the provided action. This is useful for testing smart contracts, running integration tests, or any other CI/CD workflows that need a Stellar network. @@ -443,6 +466,9 @@ The image also exposes a few other ports that most developers do not need, but a | 5432 | postgresql | database access port | | 6060 | horizon | admin port | | 6061 | stellar-rpc | admin port | +| 6062 | galexie | admin port | +| 1570 | history-archive | file server port | +| 1571 | ledger-meta-store | file server port | | 11625 | stellar-core | peer node port | | 11626 | stellar-core | main http port | | 11725 | stellar-core (horizon) | peer node port | diff --git a/common/galexie/bin/start b/common/galexie/bin/start new file mode 100755 index 000000000..29c2830f8 --- /dev/null +++ b/common/galexie/bin/start @@ -0,0 +1,7 @@ +#! /bin/bash + +set -e +set -o pipefail + +echo "starting galexie..." +exec /usr/bin/galexie append --start 2 --config-file /opt/stellar/galexie/etc/galexie.toml diff --git a/images.json b/images.json index 81d2bd2fb..8d69eb11f 100644 --- a/images.json +++ b/images.json @@ -47,6 +47,11 @@ "name": "lab", "repo": "stellar/laboratory", "ref": "main" + }, + { + "name": "galexie", + "repo": "stellar/stellar-galexie", + "ref": "galexie-v25.1.0" } ], "tests": { @@ -108,6 +113,11 @@ "name": "lab", "repo": "stellar/laboratory", "ref": "main" + }, + { + "name": "galexie", + "repo": "stellar/stellar-galexie", + "ref": "galexie-v25.1.0" } ], "tests": { @@ -175,6 +185,11 @@ "name": "lab", "repo": "stellar/laboratory", "ref": "21cc0d0b9080e664b9dbfae5713d9c7615613729" + }, + { + "name": "galexie", + "repo": "stellar/stellar-galexie", + "ref": "galexie-v25.1.0" } ], "tests": { @@ -230,6 +245,11 @@ "name": "lab", "repo": "stellar/laboratory", "ref": "main" + }, + { + "name": "galexie", + "repo": "stellar/stellar-galexie", + "ref": "main" } ], "tests": { @@ -286,6 +306,11 @@ "name": "lab", "repo": "stellar/laboratory", "ref": "main" + }, + { + "name": "galexie", + "repo": "stellar/stellar-galexie", + "ref": "main" } ], "tests": { diff --git a/local/galexie/etc/galexie.toml b/local/galexie/etc/galexie.toml new file mode 100644 index 000000000..2a574777b --- /dev/null +++ b/local/galexie/etc/galexie.toml @@ -0,0 +1,22 @@ +# Galexie Configuration for Local Network + +# Admin port configuration +admin_port = 6062 + +# Datastore Configuration +[datastore_config] +type = "Filesystem" + +[datastore_config.params] +destination_path = "/opt/stellar/ledger-meta-store/data" + +[datastore_config.schema] +ledgers_per_file = 1 +files_per_partition = 64000 + +# Stellar-core Configuration +[stellar_core_config] +network_passphrase = "__NETWORK__" +history_archive_urls = ["http://localhost:1570"] +stellar_core_binary_path = "/usr/bin/stellar-core" +captive_core_toml_path = "/opt/stellar/galexie/etc/stellar-captive-core.cfg" diff --git a/local/galexie/etc/stellar-captive-core.cfg b/local/galexie/etc/stellar-captive-core.cfg new file mode 100644 index 000000000..2deaac506 --- /dev/null +++ b/local/galexie/etc/stellar-captive-core.cfg @@ -0,0 +1,19 @@ +# Captive core configuration for galexie on local network +NETWORK_PASSPHRASE="__NETWORK__" +HTTP_PORT=11926 +PUBLIC_HTTP_PORT=false +PEER_PORT=11925 +DATABASE="sqlite3://__DATABASE__" +ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true + +UNSAFE_QUORUM=true +FAILURE_SAFETY=0 + +[[VALIDATORS]] +NAME="local_core" +HOME_DOMAIN="core.local" +# From "SDQVDISRYN2JXBS7ICL7QJAEKB3HWBJFP2QECXG7GZICAHBK4UNJCWK2" +PUBLIC_KEY="GCTI6HMWRH2QGMFKWVU5M5ZSOTKL7P7JAHZDMJJBKDHGWTEC4CJ7O3DU" +ADDRESS="localhost:11625" +QUALITY="MEDIUM" +HISTORY="curl -sf http://localhost:1570/{0} -o {1}" diff --git a/local/nginx/etc/conf.d/ledger-meta.conf b/local/nginx/etc/conf.d/ledger-meta.conf new file mode 100644 index 000000000..fe8d11063 --- /dev/null +++ b/local/nginx/etc/conf.d/ledger-meta.conf @@ -0,0 +1,5 @@ +location /ledger-meta { + rewrite /ledger-meta/(.*) /$1 break; + proxy_pass http://127.0.0.1:1571; + proxy_redirect off; +} diff --git a/local/supervisor/etc/supervisord.conf.d/galexie.conf b/local/supervisor/etc/supervisord.conf.d/galexie.conf new file mode 100644 index 000000000..754af8651 --- /dev/null +++ b/local/supervisor/etc/supervisord.conf.d/galexie.conf @@ -0,0 +1,9 @@ +[program:galexie] +user=stellar +directory=/opt/stellar/galexie +command=/opt/stellar/galexie/bin/start +autostart=false +startretries=50 +autorestart=true +priority=70 +redirect_stderr=true diff --git a/local/supervisor/etc/supervisord.conf.d/ledger-meta-store.conf b/local/supervisor/etc/supervisord.conf.d/ledger-meta-store.conf new file mode 100644 index 000000000..8ac96b62f --- /dev/null +++ b/local/supervisor/etc/supervisord.conf.d/ledger-meta-store.conf @@ -0,0 +1,9 @@ +[program:ledger-meta-store] +user=stellar +directory=/opt/stellar/ledger-meta-store/data +command=/usr/bin/python3 -m http.server 1571 +autostart=true +autorestart=true +startretries=100 +priority=10 +redirect_stderr=true diff --git a/start b/start index ee151bce0..c3de6a62b 100755 --- a/start +++ b/start @@ -16,6 +16,8 @@ export FBHOME="$STELLAR_HOME/friendbot" export LABHOME="$STELLAR_HOME/lab" export NXHOME="$STELLAR_HOME/nginx" export STELLAR_RPC_HOME="$STELLAR_HOME/stellar-rpc" +export GALEXIEHOME="$STELLAR_HOME/galexie" +export LEDGERMETASTOREHOME="$STELLAR_HOME/ledger-meta-store" export HISTORYARCHIVEHOME="$STELLAR_HOME/history-archive" export CORELOG="/var/log/stellar-core" @@ -33,6 +35,7 @@ export PROTOCOL_VERSION_DEFAULT="$(< /image.json jq -r '.config.protocol_version : "${ENABLE_CORE:=false}" : "${ENABLE_HORIZON:=false}" : "${ENABLE_LAB:=false}" +: "${ENABLE_GALEXIE:=false}" # TODO: Remove once the Soroban RPC name is fully deprecated : "${ENABLE_SOROBAN_RPC:=false}" : "${ENABLE_RPC:=$ENABLE_SOROBAN_RPC}" @@ -64,6 +67,10 @@ function validate_before_start() { echo "--randomize-network-passphrase is only supported in the local network" >&2 exit 1 fi + if [ "$NETWORK" != "local" ] && [ "$ENABLE_GALEXIE" = "true" ]; then + echo "--enable galexie is only supported in the local network" >&2 + exit 1 + fi if [ "$NETWORK" = "local" ] && [ "$DISABLE_SOROBAN_DIAGNOSTIC_EVENTS" = "false" ]; then ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true fi @@ -97,6 +104,8 @@ function start() { echo " $(< /image.json jq -r '.deps[] | select(.name == "friendbot") | "\(.ref) (\(.sha))"')" echo " lab:" echo " $(< /image.json jq -r '.deps[] | select(.name == "lab") | "\(.ref) (\(.sha))"')" + echo " galexie:" + echo " $(< /image.json jq -r '.deps[] | select(.name == "galexie") | "\(.ref) (\(.sha))"')" echo "mode: $STELLAR_MODE" echo "network: $NETWORK" @@ -115,6 +124,7 @@ function start() { init_horizon copy_pgpass init_stellar_rpc + init_galexie stop_postgres # this gets started in init_db @@ -226,6 +236,10 @@ function process_args() { ENABLE_LAB=true fi + if [[ ",$ENABLE," = *",galexie,"* ]]; then + ENABLE_GALEXIE=true + fi + case "$NETWORK" in testnet) export NETWORK_PASSPHRASE="Test SDF Network ; September 2015" @@ -329,6 +343,8 @@ function copy_defaults() { if [ "$ENABLE_RPC" = "true" ]; then cp /opt/stellar-default/$NETWORK/supervisor/etc/supervisord.conf.d/stellar-rpc.conf $SUPHOME/etc/supervisord.conf.d 2>/dev/null || true fi + cp /opt/stellar-default/$NETWORK/supervisor/etc/supervisord.conf.d/galexie.conf $SUPHOME/etc/supervisord.conf.d 2>/dev/null || true + cp /opt/stellar-default/$NETWORK/supervisor/etc/supervisord.conf.d/ledger-meta-store.conf $SUPHOME/etc/supervisord.conf.d 2>/dev/null || true cp /opt/stellar-default/$NETWORK/supervisor/etc/supervisord.conf.d/friendbot.conf $SUPHOME/etc/supervisord.conf.d 2>/dev/null || true cp /opt/stellar-default/$NETWORK/supervisor/etc/supervisord.conf.d/history-archive.conf $SUPHOME/etc/supervisord.conf.d 2>/dev/null || true fi @@ -396,6 +412,15 @@ function copy_defaults() { $CP /opt/stellar-default/$NETWORK/nginx/ $NXHOME fi fi + + if [ -d $GALEXIEHOME/etc ]; then + echo "galexie: config directory exists, skipping copy" + else + $CP /opt/stellar-default/common/galexie/ $GALEXIEHOME + if [ -d /opt/stellar-default/$NETWORK/galexie/ ]; then + $CP /opt/stellar-default/$NETWORK/galexie/ $GALEXIEHOME + fi + fi } function copy_pgpass() { @@ -604,6 +629,33 @@ function init_stellar_rpc() { popd } +function init_galexie() { + if [ "$ENABLE_GALEXIE" != "true" ]; then + return 0 + fi + + if [ -f $GALEXIEHOME/.quickstart-initialized ]; then + echo "galexie: already initialized" + return 0 + fi + + run_silent "mkdir-ledger-meta-store" mkdir -p $LEDGERMETASTOREHOME/data + run_silent "chown-ledger-meta-store" chown -R stellar:stellar $LEDGERMETASTOREHOME + + pushd $GALEXIEHOME + mkdir ./captive-core + + GALEXIE_CAPTIVE_CORE_CFG=$GALEXIEHOME/etc/stellar-captive-core.cfg + run_silent "finalize-galexie-captivecore-db" perl -pi -e "s*__DATABASE__*$GALEXIEHOME/captive-core/galexie.db*g" $GALEXIE_CAPTIVE_CORE_CFG + perl -pi -e "s/__NETWORK__/$NETWORK_PASSPHRASE/g" etc/galexie.toml + perl -pi -e "s/__NETWORK__/$NETWORK_PASSPHRASE/g" etc/stellar-captive-core.cfg + + run_silent "init-galexie" chown -R stellar:stellar . + + touch .quickstart-initialized + popd +} + function kill_supervisor() { kill -3 $(cat "/var/run/supervisord.pid") } @@ -736,6 +788,10 @@ function start_optional_services() { if [ "$ENABLE_LAB" == "true" ]; then supervisorctl start stellar-lab fi + + if [ "$ENABLE_GALEXIE" == "true" ]; then + supervisorctl start galexie + fi } function exec_supervisor() { diff --git a/tests/test_galexie.go b/tests/test_galexie.go new file mode 100644 index 000000000..5627c722f --- /dev/null +++ b/tests/test_galexie.go @@ -0,0 +1,189 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "regexp" + "strings" + "time" +) + +const ledgerMetaURL = "http://localhost:8000/ledger-meta" + +// ConfigFile represents the SEP-54 .config.json structure +type ConfigFile struct { + NetworkPassphrase string `json:"networkPassphrase"` + Version string `json:"version"` + Compression string `json:"compression"` + LedgersPerBatch int `json:"ledgersPerBatch"` + BatchesPerPartition int `json:"batchesPerPartition"` +} + +func main() { + // Wait for galexie to export some ledgers + // With ledgers_per_file=1 and files_per_partition=64000, ledger 2 would be at: + // FFFFFFFF--0-63999/FFFFFFFD--2.xdr.zst + // The partition directory format is: %08X--%d-%d (MaxUint32-partitionStart, partitionStart, partitionEnd) + // The file format is: %08X--%d.xdr.zst (MaxUint32-fileStart, fileStart) + + partitionDir := "FFFFFFFF--0-63999" + + // Test 1: Download and verify the SEP-54 .config.json file exists at the root + configURL := fmt.Sprintf("%s/.config.json", ledgerMetaURL) + logLine("Waiting for .config.json file...") + waitForConfigFile(configURL) + logLine("Config file validated!") + + // Test 2: Wait for and verify the partition directory exists + partitionURL := fmt.Sprintf("%s/%s/", ledgerMetaURL, partitionDir) + logLine("Waiting for partition directory...") + waitForURL(partitionURL) + logLine("Partition directory exists!") + + // Test 3: Wait for any ledger file to appear and download it + logLine("Waiting for ledger file...") + foundLedgerFile := waitForAnyLedgerFile(partitionURL) + if foundLedgerFile == "" { + logLine("ERROR: No ledger file found!") + os.Exit(1) + } + logLine(fmt.Sprintf("Found ledger file: %s", foundLedgerFile)) + + ledgerURL := fmt.Sprintf("%s/%s/%s", ledgerMetaURL, partitionDir, foundLedgerFile) + waitForFile(ledgerURL) + logLine("Ledger file downloaded!") + + logLine("All galexie ledger meta store tests passed!") + os.Exit(0) +} + +func waitForConfigFile(url string) { + for { + time.Sleep(5 * time.Second) + resp, err := http.Get(url) + if err != nil { + logLine(fmt.Sprintf("Waiting for config... error: %v", err)) + continue + } + + if resp.StatusCode == http.StatusOK { + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + logLine(fmt.Sprintf("Waiting for config... read error: %v", err)) + continue + } + + var config ConfigFile + if err := json.Unmarshal(body, &config); err != nil { + logLine(fmt.Sprintf("Waiting for config... JSON parse error: %v", err)) + continue + } + + // Validate required fields are present + if config.NetworkPassphrase == "" { + logLine("Waiting for config... missing networkPassphrase") + continue + } + if config.Compression == "" { + logLine("Waiting for config... missing compression") + continue + } + + return + } + resp.Body.Close() + logLine(fmt.Sprintf("Waiting for config... status: %d", resp.StatusCode)) + } +} + +func waitForURL(url string) { + for { + time.Sleep(5 * time.Second) + resp, err := http.Get(url) + if err != nil { + logLine(fmt.Sprintf("Waiting for %s... error: %v", url, err)) + continue + } + resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + return + } + logLine(fmt.Sprintf("Waiting for %s... status: %d", url, resp.StatusCode)) + } +} + +func waitForFile(url string) { + for { + time.Sleep(5 * time.Second) + resp, err := http.Get(url) + if err != nil { + logLine(fmt.Sprintf("Waiting for %s... error: %v", url, err)) + continue + } + + if resp.StatusCode == http.StatusOK { + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + logLine(fmt.Sprintf("Waiting for %s... read error: %v", url, err)) + continue + } + + if len(body) == 0 { + logLine(fmt.Sprintf("Waiting for %s... empty file", url)) + continue + } + + // For .json files, verify it looks like JSON + if strings.HasSuffix(url, ".json") { + content := string(body) + if !strings.HasPrefix(strings.TrimSpace(content), "{") { + logLine(fmt.Sprintf("Waiting for %s... not valid JSON", url)) + continue + } + } + return + } + resp.Body.Close() + logLine(fmt.Sprintf("Waiting for %s... status: %d", url, resp.StatusCode)) + } +} + +func waitForAnyLedgerFile(partitionURL string) string { + // Pattern to match ledger files like "FFFFFFFD--2.xdr.zst" + ledgerFilePattern := regexp.MustCompile(`[0-9A-Fa-f]{8}--\d+\.xdr\.zst`) + + for { + time.Sleep(5 * time.Second) + resp, err := http.Get(partitionURL) + if err != nil { + logLine(fmt.Sprintf("Waiting for ledger files... error: %v", err)) + continue + } + + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + logLine(fmt.Sprintf("Waiting for ledger files... read error: %v", err)) + continue + } + + // Find any ledger file in the listing + match := ledgerFilePattern.FindString(string(body)) + if match != "" { + return match + } + + logLine("Waiting for ledger files...") + } +} + +func logLine(text interface{}) { + log.Println("\033[32;1m[test]\033[0m", text) +}