Skip to content

Commit 9077810

Browse files
Atrax1EasterTheBunny
Atrax1
and
EasterTheBunny
authored
feat(local): add billing platform service (#1991)
* feat(local): add billing platform service * refactor * update billing with env vars and input configs * address comments * fix env var name mismatch * one more env var ref fix * static ports * clean up --------- Co-authored-by: Awbrey Hughlett <[email protected]>
1 parent 227e5cf commit 9077810

File tree

5 files changed

+476
-1
lines changed

5 files changed

+476
-1
lines changed

‎framework/components/dockercompose/billing_platform_service/billing_platform_service.go‎

Lines changed: 274 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,274 @@
1+
package billing_platform_service
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"os"
7+
"strconv"
8+
"strings"
9+
"time"
10+
11+
"github.com/docker/docker/client"
12+
"github.com/docker/go-connections/nat"
13+
"github.com/pkg/errors"
14+
"github.com/testcontainers/testcontainers-go"
15+
"github.com/testcontainers/testcontainers-go/modules/compose"
16+
"github.com/testcontainers/testcontainers-go/wait"
17+
18+
"github.com/smartcontractkit/chainlink-testing-framework/framework"
19+
"github.com/smartcontractkit/chainlink-testing-framework/framework/components/dockercompose/utils"
20+
"github.com/smartcontractkit/freeport"
21+
)
22+
23+
const DefaultPostgresDSN = "postgres://postgres:postgres@postgres:5432/billing_platform?sslmode=disable"
24+
25+
type Output struct {
26+
BillingPlatformService *BillingPlatformServiceOutput
27+
Postgres *PostgresOutput
28+
}
29+
30+
type BillingPlatformServiceOutput struct {
31+
BillingGRPCInternalURL string
32+
BillingGRPCExternalURL string
33+
CreditGRPCInternalURL string
34+
CreditGRPCExternalURL string
35+
}
36+
37+
type PostgresOutput struct {
38+
DSN string
39+
}
40+
41+
type Input struct {
42+
ComposeFile string `toml:"compose_file"`
43+
ExtraDockerNetworks []string `toml:"extra_docker_networks"`
44+
Output *Output `toml:"output"`
45+
UseCache bool `toml:"use_cache"`
46+
ChainSelector uint64 `toml:"chain_selector"`
47+
StreamsAPIURL string `toml:"streams_api_url"`
48+
StreamsAPIKey string `toml:"streams_api_key"`
49+
StreamsAPISecret string `toml:"streams_api_secret"`
50+
RPCURL string `toml:"rpc_url"`
51+
WorkflowRegistryAddress string `toml:"workflow_registry_address"`
52+
CapabilitiesRegistryAddress string `toml:"capabilities_registry_address"`
53+
WorkflowOwners []string `toml:"workflow_owners"`
54+
}
55+
56+
func defaultBillingPlatformService(in *Input) *Input {
57+
if in.ComposeFile == "" {
58+
in.ComposeFile = "./docker-compose.yml"
59+
}
60+
return in
61+
}
62+
63+
const (
64+
DEFAULT_BILLING_PLATFORM_SERVICE_BILLING_GRPC_PORT = "2222"
65+
DEFAULT_BILLING_PLATFORM_SERVICE_CREDIT_GRPC_PORT = "2223"
66+
DEFAULT_POSTGRES_PORT = "5432"
67+
DEFAULT_BILLING_PLATFORM_SERVICE_SERVICE_NAME = "billing-platform-service"
68+
DEFAULT_POSTGRES_SERVICE_NAME = "postgres"
69+
)
70+
71+
// New starts a Billing Platform Service stack using docker-compose. Various env vars are set to sensible defaults and
72+
// input values, but can be overridden by the host process env vars if needed.
73+
//
74+
// Import env vars that can be set to override defaults:
75+
// - TEST_OWNERS = comma separated list of workflow owners
76+
// - STREAMS_API_URL = URL for the Streams API; can use a mock server if needed
77+
// - STREAMS_API_KEY = API key if using a staging or prod Streams API
78+
// - STREAMS_API_SECRET = API secret if using a staging or prod Streams API
79+
func New(in *Input) (*Output, error) {
80+
if in == nil {
81+
return nil, errors.New("input is nil")
82+
}
83+
84+
if in.UseCache && in.Output != nil {
85+
return in.Output, nil
86+
}
87+
88+
in = defaultBillingPlatformService(in)
89+
identifier := framework.DefaultTCName(DEFAULT_BILLING_PLATFORM_SERVICE_SERVICE_NAME)
90+
framework.L.Debug().Str("Compose file", in.ComposeFile).
91+
Msgf("Starting Billing Platform Service stack with identifier %s",
92+
framework.DefaultTCName(DEFAULT_BILLING_PLATFORM_SERVICE_SERVICE_NAME))
93+
94+
cFilePath, fileErr := utils.ComposeFilePath(in.ComposeFile, DEFAULT_BILLING_PLATFORM_SERVICE_SERVICE_NAME)
95+
if fileErr != nil {
96+
return nil, errors.Wrap(fileErr, "failed to get compose file path")
97+
}
98+
99+
stack, stackErr := compose.NewDockerComposeWith(
100+
compose.WithStackFiles(cFilePath),
101+
compose.StackIdentifier(identifier),
102+
)
103+
if stackErr != nil {
104+
return nil, errors.Wrap(stackErr, "failed to create compose stack for Billing Platform Service")
105+
}
106+
107+
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
108+
defer cancel()
109+
110+
// Start the stackwith all environment variables from the host process
111+
// set development defaults for necessary environment variables and allow them to be overridden by the host process
112+
envVars := make(map[string]string)
113+
114+
envVars["MAINNET_WORKFLOW_REGISTRY_CHAIN_SELECTOR"] = strconv.FormatUint(in.ChainSelector, 10)
115+
envVars["MAINNET_WORKFLOW_REGISTRY_CONTRACT_ADDRESS"] = in.WorkflowRegistryAddress
116+
envVars["MAINNET_WORKFLOW_REGISTRY_RPC_URL"] = in.RPCURL
117+
envVars["MAINNET_WORKFLOW_REGISTRY_FINALITY_DEPTH"] = "0" // Instant finality on devnet
118+
envVars["KMS_PROOF_SIGNING_KEY_ID"] = "00000000-0000-0000-0000-000000000001" // provisioned via LocalStack
119+
envVars["VERIFIER_INITIAL_INTERVAL"] = "0s" // reduced to force verifier to start immediately in integration tests
120+
envVars["VERIFIER_MAXIMUM_INTERVAL"] = "1s" // reduced to force verifier to start immediately in integration tests
121+
envVars["LINKING_REQUEST_COOLDOWN"] = "0s" // reduced to force consequtive linking requests to be processed immediately in integration tests
122+
123+
envVars["MAINNET_CAPABILITIES_REGISTRY_CHAIN_SELECTOR"] = strconv.FormatUint(in.ChainSelector, 10)
124+
envVars["MAINNET_CAPABILITIES_REGISTRY_CONTRACT_ADDRESS"] = in.CapabilitiesRegistryAddress
125+
envVars["MAINNET_CAPABILITIES_REGISTRY_RPC_URL"] = in.RPCURL
126+
envVars["MAINNET_CAPABILITIES_REGISTRY_FINALITY_DEPTH"] = "10" // Arbitrary value, adjust as needed
127+
128+
envVars["TEST_OWNERS"] = strings.Join(in.WorkflowOwners, ",")
129+
envVars["STREAMS_API_URL"] = in.StreamsAPIURL
130+
envVars["STREAMS_API_KEY"] = in.StreamsAPIKey
131+
envVars["STREAMS_API_SECRET"] = in.StreamsAPISecret
132+
133+
for _, env := range os.Environ() {
134+
pair := strings.SplitN(env, "=", 2)
135+
if len(pair) == 2 {
136+
envVars[pair[0]] = pair[1]
137+
}
138+
}
139+
140+
// set these env vars after reading env vars from host
141+
port, err := freeport.Take(1)
142+
if err != nil {
143+
return nil, errors.Wrap(err, "failed to get free port for Billing Platform Service postgres")
144+
}
145+
146+
envVars["POSTGRES_PORT"] = strconv.FormatInt(int64(port[0]), 10)
147+
envVars["DEFAULT_DSN"] = DefaultPostgresDSN
148+
149+
upErr := stack.
150+
WithEnv(envVars).
151+
Up(ctx)
152+
153+
if upErr != nil {
154+
return nil, errors.Wrap(upErr, "failed to start stack for Billing Platform Service")
155+
}
156+
157+
stack.WaitForService(DEFAULT_BILLING_PLATFORM_SERVICE_SERVICE_NAME,
158+
wait.ForAll(
159+
wait.ForLog("GRPC server is live").WithPollInterval(200*time.Millisecond),
160+
wait.ForListeningPort(nat.Port(DEFAULT_BILLING_PLATFORM_SERVICE_BILLING_GRPC_PORT)),
161+
wait.ForListeningPort(nat.Port(DEFAULT_BILLING_PLATFORM_SERVICE_CREDIT_GRPC_PORT)),
162+
).WithDeadline(1*time.Minute),
163+
)
164+
165+
billingContainer, billingErr := stack.ServiceContainer(ctx, DEFAULT_BILLING_PLATFORM_SERVICE_SERVICE_NAME)
166+
if billingErr != nil {
167+
return nil, errors.Wrap(billingErr, "failed to get billing-platform-service container")
168+
}
169+
170+
postgresContainer, postgresErr := stack.ServiceContainer(ctx, DEFAULT_POSTGRES_SERVICE_NAME)
171+
if postgresErr != nil {
172+
return nil, errors.Wrap(postgresErr, "failed to get postgres container")
173+
}
174+
175+
cli, cliErr := client.NewClientWithOpts(
176+
client.FromEnv,
177+
client.WithAPIVersionNegotiation(),
178+
)
179+
if cliErr != nil {
180+
return nil, errors.Wrap(cliErr, "failed to create docker client")
181+
}
182+
defer cli.Close()
183+
184+
// so let's try to connect to a Docker network a couple of times, there must be a race condition in Docker
185+
// and even when network sandbox has been created and container is running, this call can still fail
186+
// retrying is simpler than trying to figure out how to correctly wait for the network sandbox to be ready
187+
networks := []string{framework.DefaultNetworkName}
188+
networks = append(networks, in.ExtraDockerNetworks...)
189+
190+
for _, networkName := range networks {
191+
framework.L.Debug().Msgf("Connecting billing-platform-service to %s network", networkName)
192+
connectCtx, connectCancel := context.WithTimeout(ctx, 30*time.Second)
193+
defer connectCancel()
194+
if connectErr := utils.ConnectNetwork(connectCtx, 30*time.Second, cli, billingContainer.ID, networkName, identifier); connectErr != nil {
195+
return nil, errors.Wrapf(connectErr, "failed to connect billing-platform-service to %s network", networkName)
196+
}
197+
// verify that the container is connected to framework's network
198+
inspected, inspectErr := cli.ContainerInspect(ctx, billingContainer.ID)
199+
if inspectErr != nil {
200+
return nil, errors.Wrapf(inspectErr, "failed to inspect container %s", billingContainer.ID)
201+
}
202+
203+
_, ok := inspected.NetworkSettings.Networks[networkName]
204+
if !ok {
205+
return nil, fmt.Errorf("container %s is NOT on network %s", billingContainer.ID, networkName)
206+
}
207+
208+
framework.L.Debug().Msgf("Container %s is connected to network %s", billingContainer.ID, networkName)
209+
}
210+
211+
// get hosts for billing platform service
212+
billingExternalHost, billingExternalHostErr := utils.GetContainerHost(ctx, billingContainer)
213+
if billingExternalHostErr != nil {
214+
return nil, errors.Wrap(billingExternalHostErr, "failed to get host for Billing Platform Service")
215+
}
216+
217+
// get hosts for billing platform service
218+
postgresExternalHost, postgresExternalHostErr := utils.GetContainerHost(ctx, postgresContainer)
219+
if postgresExternalHostErr != nil {
220+
return nil, errors.Wrap(postgresExternalHostErr, "failed to get host for postgres")
221+
}
222+
223+
// get mapped ports for billing platform service
224+
serviceOutput, err := getExternalPorts(ctx, billingExternalHost, billingContainer)
225+
if err != nil {
226+
return nil, errors.Wrap(err, "failed to get mapped port for Billing Platform Service")
227+
}
228+
229+
externalPostgresPort, err := utils.FindMappedPort(ctx, 20*time.Second, postgresContainer, nat.Port(DEFAULT_POSTGRES_PORT+"/tcp"))
230+
if err != nil {
231+
return nil, errors.Wrap(err, "failed to get mapped port for postgres")
232+
}
233+
234+
output := &Output{
235+
BillingPlatformService: serviceOutput,
236+
Postgres: &PostgresOutput{
237+
DSN: fmt.Sprintf("postgres://postgres:postgres@%s:%s/billing_platform", postgresExternalHost, externalPostgresPort.Port()),
238+
},
239+
}
240+
241+
framework.L.Info().Msg("Billing Platform Service stack start")
242+
243+
return output, nil
244+
}
245+
246+
func getExternalPorts(ctx context.Context, billingExternalHost string, billingContainer *testcontainers.DockerContainer) (*BillingPlatformServiceOutput, error) {
247+
ports := map[string]nat.Port{
248+
"billing": DEFAULT_BILLING_PLATFORM_SERVICE_BILLING_GRPC_PORT,
249+
"credit": DEFAULT_BILLING_PLATFORM_SERVICE_CREDIT_GRPC_PORT,
250+
}
251+
252+
output := BillingPlatformServiceOutput{}
253+
254+
for name, defaultPort := range ports {
255+
externalPort, err := utils.FindMappedPort(ctx, 20*time.Second, billingContainer, defaultPort)
256+
if err != nil {
257+
return nil, errors.Wrap(err, "failed to get mapped port for Billing Platform Service")
258+
}
259+
260+
internal := fmt.Sprintf("http://%s:%s", DEFAULT_BILLING_PLATFORM_SERVICE_SERVICE_NAME, defaultPort)
261+
external := fmt.Sprintf("http://%s:%s", billingExternalHost, externalPort.Port())
262+
263+
switch name {
264+
case "billing":
265+
output.BillingGRPCInternalURL = internal
266+
output.BillingGRPCExternalURL = external
267+
case "credit":
268+
output.CreditGRPCInternalURL = internal
269+
output.CreditGRPCExternalURL = external
270+
}
271+
}
272+
273+
return &output, nil
274+
}

‎framework/components/dockercompose/billing_platform_service/docker-compose.yml‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
services:
2+
3+
billing-platform-service:
4+
image: ${BILLING_PLATFORM_SERVICE_IMAGE:-billing-platform-service:local-cre}
5+
container_name: billing-platform-service
6+
depends_on:
7+
postgres:
8+
condition: service_healthy
9+
migrations:
10+
condition: service_started
11+
restart: on-failure
12+
tty: true
13+
command: ["grpc", "billing", "reserve"]
14+
environment:
15+
DISABLE_AUTH: true
16+
PROMETHEUS_PORT: 2112
17+
BILLING_SERVER_PORT: 2222
18+
CREDIT_RESERVATION_SERVER_PORT: 2223
19+
WORKFLOW_OWNERSHIP_PROOF_SERVER_HOST: 0.0.0.0
20+
MAINNET_WORKFLOW_REGISTRY_CHAIN_SELECTOR: ${MAINNET_WORKFLOW_REGISTRY_CHAIN_SELECTOR:-}
21+
TESTNET_WORKFLOW_REGISTRY_CHAIN_SELECTOR: ${MAINNET_WORKFLOW_REGISTRY_CHAIN_SELECTOR:-}
22+
MAINNET_WORKFLOW_REGISTRY_CONTRACT_ADDRESS: ${MAINNET_WORKFLOW_REGISTRY_CONTRACT_ADDRESS:-}
23+
TESTNET_WORKFLOW_REGISTRY_CONTRACT_ADDRESS: ${MAINNET_WORKFLOW_REGISTRY_CONTRACT_ADDRESS:-}
24+
MAINNET_WORKFLOW_REGISTRY_RPC_URL: ${MAINNET_WORKFLOW_REGISTRY_RPC_URL:-}
25+
TESTNET_WORKFLOW_REGISTRY_RPC_URL: ${MAINNET_WORKFLOW_REGISTRY_RPC_URL:-}
26+
MAINNET_WORKFLOW_REGISTRY_FINALITY_DEPTH: ${MAINNET_WORKFLOW_REGISTRY_FINALITY_DEPTH:-}
27+
KMS_PROOF_SIGNING_KEY_ID: ${KMS_PROOF_SIGNING_KEY_ID:-}
28+
VERIFIER_INITIAL_INTERVAL: ${VERIFIER_INITIAL_INTERVAL:-}
29+
VERIFIER_MAXIMUM_INTERVAL: ${VERIFIER_MAXIMUM_INTERVAL:-}
30+
LINKING_REQUEST_COOLDOWN: ${LINKING_REQUEST_COOLDOWN:-}
31+
MAINNET_CAPABILITIES_REGISTRY_CHAIN_SELECTOR: ${MAINNET_CAPABILITIES_REGISTRY_CHAIN_SELECTOR:-}
32+
TESTNET_CAPABILITIES_REGISTRY_CHAIN_SELECTOR: ${MAINNET_CAPABILITIES_REGISTRY_CHAIN_SELECTOR:-}
33+
MAINNET_CAPABILITIES_REGISTRY_CONTRACT_ADDRESS: ${MAINNET_CAPABILITIES_REGISTRY_CONTRACT_ADDRESS:-}
34+
MAINNET_CAPABILITIES_REGISTRY_RPC_URL: ${MAINNET_CAPABILITIES_REGISTRY_RPC_URL:-}
35+
TESTNET_CAPABILITIES_REGISTRY_RPC_URL: ${MAINNET_CAPABILITIES_REGISTRY_RPC_URL:-}
36+
MAINNET_CAPABILITIES_REGISTRY_FINALITY_DEPTH: ${MAINNET_CAPABILITIES_REGISTRY_FINALITY_DEPTH:-}
37+
TEST_OWNERS: ${TEST_OWNERS:-}
38+
STREAMS_API_URL: ${STREAMS_API_URL:-}
39+
STREAMS_API_KEY: ${STREAMS_API_KEY:-}
40+
STREAMS_API_SECRET: ${STREAMS_API_SECRET:-}
41+
DB_HOST: postgres
42+
DB_PORT: 5432
43+
DB_NAME: billing_platform
44+
DB_USERNAME: postgres
45+
DB_PASSWORD: postgres
46+
ports:
47+
- "2112:2112"
48+
- "2222:2222"
49+
- "2223:2223"
50+
healthcheck:
51+
test: ["CMD", "grpc_health_probe", "-addr=localhost:2222"]
52+
interval: 200ms
53+
timeout: 10s
54+
retries: 3
55+
56+
postgres:
57+
image: postgres:16-alpine
58+
container_name: postgres-billing-platform
59+
restart: always
60+
environment:
61+
POSTGRES_HOST_AUTH_METHOD: trust
62+
POSTGRES_DB: billing_platform
63+
POSTGRES_HOST: postgres
64+
POSTGRES_USER: postgres
65+
POSTGRES_PASSWORD: postgres
66+
ports:
67+
- "${POSTGRES_PORT:-5432}:5432"
68+
healthcheck:
69+
test: ["CMD","pg_isready","-U","${POSTGRES_USER}","-d","${POSTGRES_DB}","-h","${POSTGRES_HOST}"]
70+
interval: 5s
71+
timeout: 10s
72+
retries: 3
73+
74+
migrations:
75+
image: ${BILLING_PLATFORM_SERVICE_IMAGE:-billing-platform-service:local-cre}
76+
container_name: db-migrations-billing-platform
77+
depends_on:
78+
postgres:
79+
condition: service_healthy
80+
restart: on-failure
81+
command: ["db", "create-and-migrate", "--url", "${DEFAULT_DSN:-}"]
82+
83+
populate_test_data:
84+
image: ${BILLING_PLATFORM_SERVICE_IMAGE:-billing-platform-service:local-cre}
85+
container_name: populate-data-billing-platform
86+
depends_on:
87+
billing-platform-service:
88+
condition: service_started
89+
restart: on-failure
90+
command: ["db", "populate-data", "--environment=local", "--billing-client-url=billing-platform-service:2222", "--simple-linking", "--dsn=${DEFAULT_DSN:-}"]
91+
environment:
92+
TEST_OWNERS: ${TEST_OWNERS:-}
93+
MAINNET_CAPABILITIES_REGISTRY_CHAIN_SELECTOR: ${MAINNET_CAPABILITIES_REGISTRY_CHAIN_SELECTOR:-}

‎framework/components/dockercompose/go.mod‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,11 @@ require (
88
github.com/avast/retry-go/v4 v4.6.1
99
github.com/confluentinc/confluent-kafka-go v1.9.2
1010
github.com/docker/docker v28.0.4+incompatible
11+
github.com/docker/go-connections v0.5.0
1112
github.com/google/go-github/v72 v72.0.0
1213
github.com/pkg/errors v0.9.1
1314
github.com/smartcontractkit/chainlink-testing-framework/framework v0.0.0-00010101000000-000000000000
15+
github.com/smartcontractkit/freeport v0.1.2
1416
github.com/testcontainers/testcontainers-go v0.37.0
1517
github.com/testcontainers/testcontainers-go/modules/compose v0.37.0
1618
golang.org/x/oauth2 v0.25.0
@@ -66,7 +68,6 @@ require (
6668
github.com/docker/distribution v2.8.3+incompatible // indirect
6769
github.com/docker/docker-credential-helpers v0.8.2 // indirect
6870
github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect
69-
github.com/docker/go-connections v0.5.0 // indirect
7071
github.com/docker/go-metrics v0.0.1 // indirect
7172
github.com/docker/go-units v0.5.0 // indirect
7273
github.com/ebitengine/purego v0.8.2 // indirect

‎framework/components/dockercompose/go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -567,6 +567,8 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ
567567
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
568568
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA=
569569
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
570+
github.com/smartcontractkit/freeport v0.1.2 h1:xMZ0UFHmjfB4MwbDANae3RS7UKt7OJ0JVqhjPSXdKVk=
571+
github.com/smartcontractkit/freeport v0.1.2/go.mod h1:T4zH9R8R8lVWKfU7tUvYz2o2jMv1OpGCdpY2j2QZXzU=
570572
github.com/spdx/tools-golang v0.5.3 h1:ialnHeEYUC4+hkm5vJm4qz2x+oEJbS0mAMFrNXdQraY=
571573
github.com/spdx/tools-golang v0.5.3/go.mod h1:/ETOahiAo96Ob0/RAIBmFZw6XN0yTnyr/uFZm2NTMhI=
572574
github.com/spf13/cast v0.0.0-20150508191742-4d07383ffe94 h1:JmfC365KywYwHB946TTiQWEb8kqPY+pybPLoGE9GgVk=

0 commit comments

Comments
 (0)