Skip to content

Commit 3f05a10

Browse files
authored
[TT-1806] add chain.link labels documentation (#1399)
1 parent a05d532 commit 3f05a10

File tree

5 files changed

+199
-38
lines changed

5 files changed

+199
-38
lines changed

book/src/SUMMARY.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@
9191
- [Kubernetes](lib/k8s/KUBERNETES.md)
9292
- [K8s Remote Run](lib/k8s/REMOTE_RUN.md)
9393
- [K8s Tutorial](lib/k8s/TUTORIAL.md)
94+
- [k8s chain.link labels](lib/k8s/labels.md)
9495
- [Config](lib/config/config.md)
9596
- [CRIB Connector](lib/crib.md)
9697
---

book/src/lib/k8s/REMOTE_RUN.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@ export TEST_ENV_VAR=myTestVarForAJob
3939
# your image to run as a k8s job
4040
ACCOUNT=$(aws sts get-caller-identity | jq -r .Account)
4141
export ENV_JOB_IMAGE="${ACCOUNT}.dkr.ecr.us-west-2.amazonaws.com/core-integration-tests:v1.1"
42+
export DETACH_RUNNER=true # if you want the test job to run in the background after it has started
43+
export CHAINLINK_ENV_USER=yourUser # user to run the tests
44+
export CHAINLINK_USER_TEAM=yourTeam # team to run the tests for
4245
# your example test file to run inside k8s
4346
# if ENV_JOB_IMAGE is present it will create a job, wait until it finished and get logs
4447
go run examples/remote-test-runner/env.go

book/src/lib/k8s/TUTORIAL.md

Lines changed: 127 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@ func main() {
5252
err := environment.New(&environment.Config{
5353
KeepConnection: false,
5454
RemoveOnInterrupt: false,
55+
Labels: []string{"chain.link/product=myProduct", "chain.link/team=my-team", "chain.link/cost-center=test-tooling-load-test"},
56+
WorkloadLabels: map[string]string{"chain.link/product": "myProduct", "chain.link/team": "my-team", "chain.link/cost-center": "test-tooling-load-test"},
57+
PodLabels: map[string]string{"chain.link/product": "myProduct", "chain.link/team": "my-team", "chain.link/cost-center": "test-tooling-load-test"},
5558
}).
5659
AddHelm(ethereum.New(nil)).
5760
AddHelm(chainlink.New(0, nil)).
@@ -66,6 +69,11 @@ Then run `go run examples/simple/env.go`
6669

6770
Now you have your environment running, you can [connect](#connect-to-environment) to it later
6871

72+
> [!NOTE]
73+
> `chain.link/*` labels are used for internal reporting and cost allocation. They are strictly required and validated. You won't be able to create a new environment without them.
74+
> In this tutorial we create almost all of them manually, but there are convenience functions to do it for you.
75+
> You can read more about labels [here](./labels.md)
76+
6977
## Connect to environment
7078

7179
We've already created an environment [previously](#getting-started), now we can connect
@@ -84,7 +92,7 @@ You can get the namespace name from logs on creation time
8492

8593
## Debugging a new integration environment
8694

87-
You can spin up environment and block on forwarder if you'd like to run some other code
95+
You can spin up environment and block on forwarder if you'd like to run some other code. Let's use convenience functions for creating `chain.link` labels.
8896

8997
```golang
9098
package main
@@ -96,8 +104,17 @@ import (
96104
)
97105

98106
func main() {
107+
nsLabels, err := GetRequiredChainLinkNamespaceLabels("my-product", "load")
108+
require.NoError(t, err, "Error creating required chain.link labels for namespace")
109+
110+
workloadPodLabels, err := GetRequiredChainLinkWorkloadAndPodLabels("my-product", "load")
111+
require.NoError(t, err, "Error creating required chain.link labels for workloads and pods")
112+
113+
nsLabels := append(nsLabel,s "type=construction-in-progress")
99114
err := environment.New(&environment.Config{
100-
Labels: []string{"type=construction-in-progress"},
115+
Labels: nsLabels,
116+
WorkloadLabels: workloadPodLabels
117+
PodLabels: workloadPodLabels
101118
NamespacePrefix: "new-environment",
102119
KeepConnection: true,
103120
RemoveOnInterrupt: true,
@@ -134,6 +151,8 @@ type ConnectedChart interface {
134151
GetValues() *map[string]any
135152
// ExportData export deployment part data in the env
136153
ExportData(e *Environment) error
154+
// GetLabels get labels for component, it must return `chain.link/component` label
155+
GetLabels() map[string]string
137156
}
138157
```
139158

@@ -153,6 +172,12 @@ func New(props *Props) environment.ConnectedChart {
153172
Props: props,
154173
}
155174
}
175+
176+
func (m NewDeploymentPart) GetLabels() map[string]string {
177+
return map[string]string{
178+
"chain.link/component": "new-deployment-part",
179+
}
180+
}
156181
```
157182

158183
Now let's tie them together
@@ -169,6 +194,9 @@ import (
169194

170195
func main() {
171196
e := environment.New(&environment.Config{
197+
Labels: []string{"chain.link/product=myProduct", "chain.link/team=my-team", "chain.link/cost-center=test-tooling-load-test"},
198+
WorkloadLabels: map[string]string{"chain.link/product": "myProduct", "chain.link/team": "my-team", "chain.link/cost-center": "test-tooling-load-test"},
199+
PodLabels: map[string]string{"chain.link/product": "myProduct", "chain.link/team": "my-team", "chain.link/cost-center": "test-tooling-load-test"},
172200
NamespacePrefix: "adding-new-deployment-part",
173201
TTL: 3 * time.Hour,
174202
KeepConnection: true,
@@ -220,6 +248,8 @@ type ConnectedChart interface {
220248
GetValues() *map[string]any
221249
// ExportData export deployment part data in the env
222250
ExportData(e *Environment) error
251+
// GetLabels get labels for component, it must return `chain.link/component` label
252+
GetLabels() map[string]string
223253
}
224254
```
225255

@@ -269,7 +299,12 @@ import (
269299
)
270300

271301
func main() {
272-
e := environment.New(nil)
302+
envConfig := &environment.Config{
303+
Labels: []string{"chain.link/product=myProduct", "chain.link/team=my-team", "chain.link/cost-center=test-tooling-load-test"},
304+
WorkloadLabels: map[string]string{"chain.link/product": "myProduct", "chain.link/team": "my-team", "chain.link/cost-center": "test-tooling-load-test"},
305+
PodLabels: map[string]string{"chain.link/product": "myProduct", "chain.link/team": "my-team", "chain.link/cost-center": "test-tooling-load-test"}
306+
}
307+
e := environment.New(envConfig)
273308
err := e.
274309
AddChart(blockscout.New(&blockscout.Props{})). // you can also add cdk8s charts if you like Go code
275310
AddHelm(ethereum.New(nil)).
@@ -322,10 +357,13 @@ import (
322357
)
323358

324359
func main() {
325-
e := environment.New(&environment.Config{
326-
NamespacePrefix: "modified-env",
327-
Labels: []string{fmt.Sprintf("envType=Modified")},
328-
}).
360+
modifiedEnvConfig := &environment.Config{
361+
NamespacePrefix: "modified-env",
362+
Labels: []string{"envType=Modified", "chain.link/product=myProduct", "chain.link/team=my-team", "chain.link/cost-center=test-tooling-load-test"},
363+
WorkloadLabels: map[string]string{"chain.link/product": "myProduct", "chain.link/team": "my-team", "chain.link/cost-center": "test-tooling-load-test"},
364+
PodLabels: map[string]string{"chain.link/product": "myProduct", "chain.link/team": "my-team", "chain.link/cost-center": "test-tooling-load-test"}
365+
}
366+
e := environment.New(modifiedEnvConfig).
329367
AddChart(blockscout.New(&blockscout.Props{
330368
WsURL: "ws://geth:8546",
331369
HttpURL: "http://geth:8544",
@@ -371,16 +409,19 @@ import (
371409
)
372410

373411
func main() {
374-
e := environment.New(&environment.Config{
375-
NamespacePrefix: "modified-env",
376-
Labels: []string{fmt.Sprintf("envType=Modified")},
377-
}).
378-
AddHelm(mockservercfg.New(nil)).
379-
AddHelm(mockserver.New(nil)).
380-
AddHelm(ethereum.New(nil)).
381-
AddHelm(chainlink.New(0, map[string]any{
382-
"replicas": 1,
383-
}))
412+
modifiedEnvConfig := &environment.Config{
413+
NamespacePrefix: "modified-env",
414+
Labels: []string{"envType=Modified", "chain.link/product=myProduct", "chain.link/team=my-team", "chain.link/cost-center=test-tooling-load-test"},
415+
WorkloadLabels: map[string]string{"chain.link/product": "myProduct", "chain.link/team": "my-team", "chain.link/cost-center": "test-tooling-load-test"},
416+
PodLabels: map[string]string{"chain.link/product": "myProduct", "chain.link/team": "my-team", "chain.link/cost-center": "test-tooling-load-test"}
417+
}
418+
e := environment.New(modifiedEnvConfig).
419+
AddHelm(mockservercfg.New(nil)).
420+
AddHelm(mockserver.New(nil)).
421+
AddHelm(ethereum.New(nil)).
422+
AddHelm(chainlink.New(0, map[string]any{
423+
"replicas": 1,
424+
}))
384425
err := e.Run()
385426
if err != nil {
386427
panic(err)
@@ -427,6 +468,10 @@ const (
427468
EnvVarUserDescription = "Owner of an environment"
428469
EnvVarUserExample = "Satoshi"
429470

471+
EnvVarTeam = "CHAINLINK_USER_TEAM"
472+
EnvVarTeamDescription = "Team to, which owner of the environment belongs to"
473+
EnvVarTeamExample = "BIX, CCIP, BCM"
474+
430475
EnvVarCLCommitSha = "CHAINLINK_COMMIT_SHA"
431476
EnvVarCLCommitShaDescription = "The sha of the commit that you're running tests on. Mostly used for CI"
432477
EnvVarCLCommitShaExample = "${{ github.sha }}"
@@ -466,22 +511,55 @@ type Config struct {
466511
Namespace string
467512
// Labels is a set of labels applied to the namespace in a format of "key=value"
468513
Labels []string
469-
nsLabels *map[string]*string
470-
// ReadyCheckData is settings for readiness probes checks for all deployment components
471-
// checking that all pods are ready by default with 8 minutes timeout
472-
// &client.ReadyCheckData{
473-
// ReadinessProbeCheckSelector: "",
474-
// Timeout: 15 * time.Minute,
475-
// }
476-
ReadyCheckData *client.ReadyCheckData
477-
// DryRun if true, app will just generate a manifest in local dir
478-
DryRun bool
479-
// InsideK8s used for long-running soak tests where you connect to env from the inside
480-
InsideK8s bool
481-
// KeepConnection keeps connection until interrupted with a signal, useful when prototyping and debugging a new env
482-
KeepConnection bool
483-
// RemoveOnInterrupt automatically removes an environment on interrupt
484-
RemoveOnInterrupt bool
514+
// PodLabels is a set of labels applied to every pod in the namespace
515+
PodLabels map[string]string
516+
// WorkloadLabels is a set of labels applied to every workload in the namespace
517+
WorkloadLabels map[string]string
518+
// PreventPodEviction if true sets a k8s annotation safe-to-evict=false to prevent pods from being evicted
519+
// Note: This should only be used if your test is completely incapable of handling things like K8s rebalances without failing.
520+
// If that is the case, it's worth the effort to make your test fault-tolerant soon. The alternative is expensive and infuriating.
521+
PreventPodEviction bool
522+
// Allow deployment to nodes with these tolerances
523+
Tolerations []map[string]string
524+
// Restrict deployment to only nodes matching a particular node role
525+
NodeSelector map[string]string
526+
// ReadyCheckData is settings for readiness probes checks for all deployment components
527+
// checking that all pods are ready by default with 8 minutes timeout
528+
// &client.ReadyCheckData{
529+
// ReadinessProbeCheckSelector: "",
530+
// Timeout: 15 * time.Minute,
531+
// }
532+
ReadyCheckData *client.ReadyCheckData
533+
// DryRun if true, app will just generate a manifest in local dir
534+
DryRun bool
535+
// InsideK8s used for long-running soak tests where you connect to env from the inside
536+
InsideK8s bool
537+
// SkipManifestUpdate will skip updating the manifest upon connecting to the environment. Should be true if you wish to update the manifest (e.g. upgrade pods)
538+
SkipManifestUpdate bool
539+
// KeepConnection keeps connection until interrupted with a signal, useful when prototyping and debugging a new env
540+
KeepConnection bool
541+
// RemoveOnInterrupt automatically removes an environment on interrupt
542+
RemoveOnInterrupt bool
543+
// UpdateWaitInterval an interval to wait for deployment update started
544+
UpdateWaitInterval time.Duration
545+
546+
// Remote Runner Specific Variables //
547+
// JobImage an image to run environment as a job inside k8s
548+
JobImage string
549+
// Specify only if you want remote-runner to start with a specific name
550+
RunnerName string
551+
// Specify only if you want to mount reports from test run in remote runner
552+
ReportPath string
553+
// JobLogFunction a function that will be run on each log
554+
JobLogFunction func(*Environment, string)
555+
// Test the testing library current Test struct
556+
Test *testing.T
557+
// jobDeployed used to limit us to 1 remote runner deploy
558+
jobDeployed bool
559+
// detachRunner should we detach the remote runner after starting the test
560+
detachRunner bool
561+
// fundReturnFailed the status of a fund return
562+
fundReturnFailed bool
485563
}
486564
```
487565

@@ -501,7 +579,11 @@ import (
501579
)
502580

503581
func main() {
504-
e := environment.New(nil).
582+
e := environment.New(&environment.Config{
583+
Labels: []string{"chain.link/product=myProduct", "chain.link/team=my-team", "chain.link/cost-center=test-tooling-load-test"},
584+
WorkloadLabels: map[string]string{"chain.link/product": "myProduct", "chain.link/team": "my-team", "chain.link/cost-center": "test-tooling-load-test"},
585+
PodLabels: map[string]string{"chain.link/product": "myProduct", "chain.link/team": "my-team", "chain.link/cost-center": "test-tooling-load-test"}
586+
}).
505587
AddHelm(ethereum.New(nil)).
506588
AddHelm(chainlink.New(0, nil))
507589
if err := e.Run(); err != nil {
@@ -530,9 +612,11 @@ import (
530612
)
531613

532614
func main() {
533-
e := environment.New(&environment.Config{
534-
Labels: []string{fmt.Sprintf("envType=%s", pkg.EnvTypeEVM5)},
535-
}).
615+
e := environment.New(&environment.Config{
616+
Labels: []string{fmt.Sprintf("envType=%s", pkg.EnvTypeEVM5), "chain.link/product=myProduct", "chain.link/team=my-team", "chain.link/cost-center=test-tooling-load-test"},
617+
WorkloadLabels: map[string]string{"chain.link/product": "myProduct", "chain.link/team": "my-team", "chain.link/cost-center": "test-tooling-load-test"},
618+
PodLabels: map[string]string{"chain.link/product": "myProduct", "chain.link/team": "my-team", "chain.link/cost-center": "test-tooling-load-test"}
619+
}).
536620
AddHelm(ethereum.New(nil)).
537621
AddHelm(chainlink.New(0, nil))
538622
err := e.Run()
@@ -583,7 +667,12 @@ import (
583667
)
584668

585669
func main() {
586-
e := environment.New(nil).
670+
envConfig := &environment.Config{
671+
Labels: []string{"chain.link/product=myProduct", "chain.link/team=my-team", "chain.link/cost-center=test-tooling-load-test"},
672+
WorkloadLabels: map[string]string{"chain.link/product": "myProduct", "chain.link/team": "my-team", "chain.link/cost-center": "test-tooling-load-test"},
673+
PodLabels: map[string]string{"chain.link/product": "myProduct", "chain.link/team": "my-team", "chain.link/cost-center": "test-tooling-load-test"}
674+
}
675+
e := environment.New(envConfig).
587676
AddChart(goc.New()).
588677
AddChart(dummy.New())
589678
if err := e.Run(); err != nil {

book/src/lib/k8s/labels.md

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# k8s `chain.link` Labels
2+
3+
## Purpose
4+
Resource labeling has been introduced to better associate Kubernetes (k8s) costs with products and teams. This document describes the labels used in the k8s cluster.
5+
6+
## Required Labels
7+
Labels should be applied to all resources in the k8s cluster at three levels:
8+
- **Namespace**
9+
- **Workload**
10+
- **Pod**
11+
12+
All three levels should include the following labels:
13+
- `chain.link/team` - Name of the team that owns the resource.
14+
- `chain.link/product` - Product that the resource belongs to.
15+
- `chain.link/cost-center` - Product and framework name.
16+
17+
Additionally, pods should include the following label:
18+
- `chain.link/component` - Name of the component.
19+
20+
### `chain.link/team`
21+
This label represents the team responsible for the resource, but it might not be the team of the individual who created the resource. It should reflect the team the environment is **created for**.
22+
23+
For example, if you are a member of the Test Tooling team, but someone from the BIX team requests load tests, the namespace should be labeled as: `chain.link/team: bix`.
24+
25+
### `chain.link/product`
26+
This label specifies the product the resource belongs to. Internally, some products may have alternative names (e.g., OCR instead of Data Feeds). To standardize data analysis, use the following names:
27+
28+
```
29+
automation
30+
bcm
31+
ccip
32+
data-feedsv1.0
33+
data-feedsv2.0
34+
data-feedsv3.0
35+
data-streamsv0.3
36+
data-streamsv1.0
37+
deco
38+
functions
39+
proof-of-reserve
40+
scale
41+
staking
42+
vrf
43+
```
44+
45+
For example:
46+
- OCR version 1: `data-feedsv1.0`
47+
- OCR version 2: `data-feedsv2.0`
48+
49+
### `chain.link/cost-center`
50+
This label serves as an umbrella for specific test or environment types and should rarely change. For load or soak tests using solutions provided by the Test Tooling team, use the convention: `test-tooling-<test-type>-test`
51+
52+
For example: `test-tooling-load-test`.
53+
54+
This allows easy distinction from load tests run using other tools.
55+
56+
### `chain.link/component`
57+
This label identifies different components within the same product. Examples include:
58+
- `chainlink` - Chainlink node.
59+
- `geth` - Go-Ethereum blockchain node.
60+
- `test-runner` - Remote test runner.
61+
62+
## Adding Labels to New Components
63+
Adding a new component to an existing framework is discouraged. The recommended approach is to add the component to CRIB and make these labels part of the deployment templates.
64+
65+
If you need to add a new component, refer to the following sections in the [k8s Tutorial](./TUTORIAL.md):
66+
- **Creating a new deployment part in Helm**
67+
- **Creating a new deployment part in cdk8s**

book/src/libs/wasp/k8s.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# k8s

0 commit comments

Comments
 (0)