services/horizon/docker: cdp enabled verify range image (#5736)

sreuland · web-flow · commit a356b55b5396 · 2025-07-03T11:55:45.000-07:00
diff --git a/.github/workflows/horizon.yml b/.github/workflows/horizon.yml
@@ -124,8 +124,7 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       GO_VERSION: 1.23.4
-      STELLAR_CORE_VERSION: 22.3.0-2485.e643061a4.focal
-      CAPTIVE_CORE_STORAGE_PATH: /tmp
+      STELLAR_CORE_VERSION: 22.3.0-2485.e643061a4.jammy
     steps:
       - uses: actions/checkout@v3
         with:
@@ -134,7 +133,7 @@ jobs:
 
       - name: Build and test the Verify Range Docker image
         run: |
-          docker build --build-arg="GO_VERSION=$GO_VERSION" -f services/horizon/docker/verify-range/Dockerfile -t stellar/horizon-verify-range services/horizon/docker/verify-range/
+          docker build --build-arg="GO_VERSION=$GO_VERSION" --build-arg="STELLAR_CORE_VERSION=$STELLAR_CORE_VERSION" -f services/horizon/docker/verify-range/Dockerfile -t stellar/horizon-verify-range services/horizon/docker/verify-range/
           # Any range should do for basic testing, this range was chosen pretty early in history so that it only takes a few mins to run
           docker run -e BRANCH=$(git rev-parse HEAD) -e FROM=10000063 -e TO=10000127 stellar/horizon-verify-range
 
diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go
@@ -44,16 +44,14 @@ var (
 	parallelWorkers          uint
 	retries                  uint
 	retryBackoffSeconds      uint
-	ledgerBackendStr         string
 	storageBackendConfigPath string
 	ledgerBackendType        ingest.LedgerBackendType
 )
 
 // generateLedgerBackendOpt creates a reusable ConfigOption for ledgerbackend parameter
-func generateLedgerBackendOpt(configKey *string, backendTypeVar *ingest.LedgerBackendType) *support.ConfigOption {
+func generateLedgerBackendOpt(backendTypeVar *ingest.LedgerBackendType) *support.ConfigOption {
 	return &support.ConfigOption{
 		Name:        "ledgerbackend",
-		ConfigKey:   configKey,
 		OptType:     types.String,
 		Required:    false,
 		FlagDefault: ingest.CaptiveCoreBackend.String(),
@@ -70,7 +68,6 @@ func generateLedgerBackendOpt(configKey *string, backendTypeVar *ingest.LedgerBa
 			default:
 				return fmt.Errorf("invalid ledger backend: %s, must be 'captive-core' or 'datastore'", val)
 			}
-			*co.ConfigKey.(*string) = val
 			return nil
 		},
 	}
@@ -160,14 +157,8 @@ func ingestRangeCmdOpts() support.ConfigOptions {
 			FlagDefault: uint(5),
 			Usage:       "[optional] backoff seconds between reingest retries",
 		},
-		generateLedgerBackendOpt(&ledgerBackendStr, &ledgerBackendType),
-		{
-			Name:      "datastore-config",
-			ConfigKey: &storageBackendConfigPath,
-			OptType:   types.String,
-			Required:  false,
-			Usage:     "[optional] Specify the path to the datastore config file (required for datastore backend)",
-		},
+		generateLedgerBackendOpt(&ledgerBackendType),
+		generateDatastoreConfigOpt(&storageBackendConfigPath),
 	}
 }
 
diff --git a/services/horizon/cmd/ingest.go b/services/horizon/cmd/ingest.go
@@ -24,7 +24,6 @@ var ingestBuildStateSequence uint32
 var ingestBuildStateSkipChecks bool
 var ingestVerifyFrom, ingestVerifyTo, ingestVerifyDebugServerPort uint32
 var ingestVerifyState bool
-var ingestVerifyLedgerBackendStr string
 var ingestVerifyStorageBackendConfigPath string
 var ingestVerifyLedgerBackendType ingest.LedgerBackendType
 var processVerifyRangeFn = processVerifyRange
@@ -81,14 +80,8 @@ var ingestVerifyRangeCmdOpts = support.ConfigOptions{
 		FlagDefault: uint32(0),
 		Usage:       "[optional] opens a net/http/pprof server at given port",
 	},
-	generateLedgerBackendOpt(&ingestVerifyLedgerBackendStr, &ingestVerifyLedgerBackendType),
-	{
-		Name:      "datastore-config",
-		ConfigKey: &ingestVerifyStorageBackendConfigPath,
-		OptType:   types.String,
-		Required:  false,
-		Usage:     "[optional] Specify the path to the datastore config file (required for datastore backend)",
-	},
+	generateLedgerBackendOpt(&ingestVerifyLedgerBackendType),
+	generateDatastoreConfigOpt(&ingestVerifyStorageBackendConfigPath),
 }
 
 var stressTestNumTransactions, stressTestChangesPerTransaction int
@@ -130,10 +123,6 @@ func DefineIngestCommands(rootCmd *cobra.Command, horizonConfig *horizon.Config,
 				return err
 			}
 
-			if err := horizon.ApplyFlags(horizonConfig, horizonFlags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false}); err != nil {
-				return err
-			}
-
 			if ingestVerifyDebugServerPort != 0 {
 				go func() {
 					log.Infof("Starting debug server at: %d", ingestVerifyDebugServerPort)
@@ -157,6 +146,7 @@ func DefineIngestCommands(rootCmd *cobra.Command, horizonConfig *horizon.Config,
 			}
 
 			storageBackendConfig := ingest.StorageBackendConfig{}
+			noCaptiveCore := false
 			if ingestVerifyLedgerBackendType == ingest.BufferedStorageBackend {
 				if ingestVerifyStorageBackendConfigPath == "" {
 					return fmt.Errorf("datastore-config file path is required with datastore backend")
@@ -165,8 +155,12 @@ func DefineIngestCommands(rootCmd *cobra.Command, horizonConfig *horizon.Config,
 				if storageBackendConfig, err = loadStorageBackendConfig(ingestVerifyStorageBackendConfigPath); err != nil {
 					return err
 				}
+				noCaptiveCore = true
 			}
 
+			if err := horizon.ApplyFlags(horizonConfig, horizonFlags, horizon.ApplyOptions{NoCaptiveCore: noCaptiveCore, RequireCaptiveCoreFullConfig: false}); err != nil {
+				return err
+			}
 			err := processVerifyRangeFn(horizonConfig, horizonFlags, storageBackendConfig)
 			if err != nil {
 				return err
@@ -396,3 +390,14 @@ func processVerifyRange(horizonConfig *horizon.Config, horizonFlags config.Confi
 		ingestVerifyState,
 	)
 }
+
+// generateDatastoreConfigOpt returns a *support.ConfigOption for the datastore-config flag
+func generateDatastoreConfigOpt(configKey *string) *support.ConfigOption {
+	return &support.ConfigOption{
+		Name:      "datastore-config",
+		ConfigKey: configKey,
+		OptType:   types.String,
+		Required:  false,
+		Usage:     "[optional] Specify the path to the datastore config file (required for datastore backend)",
+	}
+}
diff --git a/services/horizon/docker/verify-range/README.md b/services/horizon/docker/verify-range/README.md
@@ -2,6 +2,11 @@
 
 This docker image allows running multiple instances of `horizon ingest verify-command` on a single machine or running it in [AWS Batch](https://aws.amazon.com/batch/).
 
+## Data directory
+The image by default stores all outputs of db, captive core, and buffered storage processes
+under the `/data` directory at runtime in the container. Therefore it is strongly recommended
+to provide an external volume mount to the container for `/data` of at least 300-500GB. Specify this at docker run time via - `docker run -v /host/volume:/data`
+
 ## Env variables
 
 ### Running locally
@@ -14,16 +19,51 @@ This docker image allows running multiple instances of `horizon ingest verify-co
 
 ### Running in AWS Batch
 
-| Name                 | Description                                                          |
-|----------------------|----------------------------------------------------------------------|
-| `BRANCH`             | Git branch to build (useful for testing PRs)                         |
-| `BATCH_START_LEDGER` | First ledger of the AWS Batch Job, must be a checkpoint ledger or 1. |
-| `BATCH_SIZE`         | Size of the batch, must be multiple of 64.                           |
+| Name                        | Description                                                          |
+|-----------------------------|----------------------------------------------------------------------|
+| `AWS_BATCH_JOB_ARRAY_INDEX` | The zero based index of a single batch Job.                          |
+| `BATCH_START_LEDGER`        | The `FROM` ledger of the requested ledger range to verify.           |
+| `BATCH_SIZE`                | Size of the batch, must be multiple of 64.                           |
+
+
+### Datastore and GCP Credentials Usage
+
+This image supports connecting to GCS buckets for ledger data instead of captive core. To use this feature configure the container with these additional settings:
 
-#### Example
+#### GCP Credentials
+- Purpose: To access GCS buckets the image needs GCP credentials. 
+- Two options are available to provide this to container:
+  - As an environment variable:
+    - Pass the GCP JSON credentials as a string in a `GCP_CREDS` environment variable:
+      ```sh
+      docker run -e GCP_CREDS='{...}' ...
+      ```
+  - As a volume mount:
+    - Mount the GCP json credentials file on host to the container, e.g.:
+      ```sh
+      docker run -v /host/path/credentials.json:/tmp/credentials.json -e GOOGLE_APPLICATION_CREDENTIALS=/tmp/credentials.json ...
+      ```
 
-When you start 10 jobs with `BATCH_START_LEDGER=63` and `BATCH_SIZE=64`
-it will run the following ranges:
+#### GCS Datastore settings
+- Purpose: Defines the GCS bucket name and ledger partioning used on the buckets. These settings are referenced as a single toml file at runtime. Here is an example [datastore_config.toml](../../../galexie/config.example.toml)
+- Two options are available to provide this to container:
+  - As an environment variable:
+    - Pass the datastore TOML config as a string(including line breaks, tabs) in the `DATASTORE_CONFIG_PLAIN` environment variable:
+      ```sh
+      docker run -e DATASTORE_CONFIG_PLAIN='[buffered_storage_backend_config]\nbuffer_size = 5\n ...' 
+      ```
+  - As a volume mount:
+    - Mount the datastore toml config file from host to the container, e.g.:
+      ```sh
+      docker run -v /host/path/datastore-config.toml:/tmp/datastore-config.toml -e DATASTORE_CONFIG=/tmp/datastore-config.toml 
+      ```
+
+### Examples of running container
+
+#### Batch jobs example
+When run from aws batch, given `BATCH_START_LEDGER=63` and `BATCH_SIZE=64`
+it will generate runner jobs and give them each a  `AWS_BATCH_JOB_ARRAY_INDEX`. 
+The verify-range container will then generate the associated ledger ranges per each job:
 
 | `AWS_BATCH_JOB_ARRAY_INDEX` | `FROM` | `TO` |
 |-----------------------------|--------|------|
@@ -32,13 +72,25 @@ it will run the following ranges:
 | 2                           | 191    | 255  |
 | 3                           | 255    | 319  |
 
-## Tips when using AWS Batch
-
-* In "Job definition" set vCPUs to 2 and Memory to 4096. This represents the `c5.large` instances Horizon should be using.
-* In "Compute environments":
-    * Set instance type to "c5.large".
-    * Set "Maximum vCPUs" to 2x the number of instances you want to start (because "c5.large" has 2 vCPUs). Ex. 10 vCPUs = 5 x "c5.large" instances.
-* Use spot instances! It's much cheaper and speed of testing will be the same in 99% of cases.
-* You need to publish the image if there are any changes in `Dockerfile` or one of the scripts.
-* When batch processing is over check if instances have been terminated. Sometimes AWS doesn't terminate them.
-* Make sure the job timeout is set to a larger value if you verify larger ranges. Default is just 100 seconds.
+#### `docker run` example
+Running the verify-range image as local container with `docker run`
+* run verify range and use captive core to get ledgers from pubnet:
+  ```
+  docker run -e FROM=63 \
+                      -e TO=127 \
+                      -e BRANCH=<target version> \
+                      -e BASE_BRANCH=master \
+                      verify-range:latest
+  ```  
+* run verify range with gcs datastore to use precomputed ledger metadata from buckets, captive core is not used:
+  ```
+  docker run -e FROM=63 \
+                      -e TO=127 \
+                      -e BRANCH=<target version> \
+                      -e BASE_BRANCH=master \
+                      -v /host/path/to/gcreds.json:/tmp/gcp.json \
+                      -e GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcp.json \
+                      -v /host/path/to/datastore-config.tom:/tmp/datastore-config.toml \
+                      -e DATASTORE_CONFIG=/tmp/datastore-config.toml \
+                      verify-range:latest
+  ```  
diff --git a/services/horizon/docker/verify-range/start b/services/horizon/docker/verify-range/start
