Skip to content

Commit c8225b6

Browse files
vaikasvaikas
committed
WIP: Add scaffolding to stand up Trillian on k8s.
Signed-off-by: Ville Aikas <vaikas@chainguard.dev>
1 parent 3016080 commit c8225b6

File tree

17 files changed

+704
-14
lines changed

17 files changed

+704
-14
lines changed

‎.github/workflows/kind-scaffolding-e2e-createtree.yml‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
name: Kind scaffolding test
2+
3+
on:
4+
pull_request:
5+
branches: [ 'main', 'release-*' ]
6+
7+
permissions: read-all
8+
9+
jobs:
10+
e2e-tests:
11+
name: test tree creation with scaffolding
12+
runs-on: ubuntu-latest
13+
14+
strategy:
15+
fail-fast: false # Keep running if one leg fails.
16+
matrix:
17+
k8s-version:
18+
- v1.22.x
19+
- v1.23.x
20+
21+
env:
22+
REGISTRY_NAME: registry.local
23+
REGISTRY_PORT: 5000
24+
KO_DOCKER_REPO: registry.local:5000/trillian
25+
26+
steps:
27+
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2.4.0
28+
- uses: actions/setup-go@b22fbbc2921299758641fab08929b4ac52b32923 # v2.2.0
29+
with:
30+
go-version: '1.17'
31+
check-latest: true
32+
33+
- uses: imjasonh/setup-ko@2c3450ca27f6e6f2b02e72a40f2163c281a1f675 # v0.4
34+
35+
- name: Install yq
36+
uses: mikefarah/yq@70403375d7b96075bd68b40c434807cff1593568 # v4.25.1
37+
38+
- name: Setup mirror
39+
uses: chainguard-dev/actions/setup-mirror@main
40+
with:
41+
mirror: mirror.gcr.io
42+
43+
- name: Setup kind cluster
44+
uses: chainguard-dev/actions/setup-kind@main
45+
with:
46+
k8s-version: ${{ matrix.k8s-version }}
47+
cluster-suffix: c${{ github.run_id }}.local
48+
49+
- name: Setup knative
50+
uses: chainguard-dev/actions/setup-knative@main
51+
with:
52+
k8s-version: ${{ matrix.k8s-version }}
53+
cluster-suffix: c${{ github.run_id }}.local
54+
55+
- name: Install Trillian
56+
run: |
57+
echo '::group:: install Trillian scaffolding'
58+
ko apply -BRf ./examples/deployment/kubernetes/scaffolding
59+
echo '::endgroup::'
60+
echo '::group:::' waiting for services to come up
61+
kubectl wait -n trillian-system --for=condition=Ready --timeout=5m ksvc --all
62+
echo '::endgroup::'
63+
64+
- name: Create a tree on it
65+
run: |
66+
echo '::group:: install create tree job'
67+
kubectl apply -Rf ./examples/deployment/kubernetes/createtree
68+
echo '::endgroup::'
69+
echo '::group:::' waiting for job to complete'
70+
kubectl wait -n createtree --for=condition=Complete --timeout=5m jobs createtree
71+
echo '::endgroup::'
72+
kubectl -n createtree get cm trillian-tree -ojsonpath='{.data.treeID}'
73+
74+
- name: Collect diagnostics
75+
if: ${{ failure() }}
76+
uses: chainguard-dev/actions/kind-diag@84c993eaf02da1c325854fb272a4df9184bd80fc # main

‎cmd/createtree-k8s/main.go‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
// Copyright 2022 The Sigstore Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package main
16+
17+
import (
18+
"context"
19+
"flag"
20+
"fmt"
21+
"time"
22+
23+
"github.com/google/trillian"
24+
"github.com/google/trillian/client"
25+
"github.com/google/trillian/client/rpcflags"
26+
"github.com/pkg/errors"
27+
"google.golang.org/grpc"
28+
"google.golang.org/protobuf/types/known/durationpb"
29+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
30+
"k8s.io/client-go/kubernetes"
31+
"k8s.io/client-go/rest"
32+
"knative.dev/pkg/logging"
33+
"knative.dev/pkg/signals"
34+
"sigs.k8s.io/release-utils/version"
35+
)
36+
37+
const (
38+
// Key in the configmap holding the value of the tree.
39+
treeKey = "treeID"
40+
)
41+
42+
var (
43+
ns = flag.String("namespace", "", "Namespace where to update the configmap in")
44+
cmname = flag.String("configmap", "", "Name of the configmap where the treeID lives")
45+
adminServerAddr = flag.String("admin_server", "log-server.trillian-system.svc:80", "Address of the gRPC Trillian Admin Server (host:port)")
46+
treeState = flag.String("tree_state", trillian.TreeState_ACTIVE.String(), "State of the new tree")
47+
treeType = flag.String("tree_type", trillian.TreeType_LOG.String(), "Type of the new tree")
48+
displayName = flag.String("display_name", "", "Display name of the new tree")
49+
description = flag.String("description", "", "Description of the new tree")
50+
maxRootDuration = flag.Duration("max_root_duration", time.Hour, "Interval after which a new signed root is produced despite no submissions; zero means never")
51+
force = flag.Bool("force", false, "Force create a new tree and update configmap")
52+
)
53+
54+
func main() {
55+
flag.Parse()
56+
ctx := signals.NewContext()
57+
if *ns == "" {
58+
logging.FromContext(ctx).Fatal("Need to specify --namespace for where to update the configmap in")
59+
}
60+
if *cmname == "" {
61+
logging.FromContext(ctx).Fatal("Need to specify --configmap for which configmap to update")
62+
}
63+
64+
versionInfo := version.GetVersionInfo()
65+
logging.FromContext(ctx).Infof("running Version: %s GitCommit: %s BuildDate: %s", versionInfo.GitVersion, versionInfo.GitCommit, versionInfo.BuildDate)
66+
67+
config, err := rest.InClusterConfig()
68+
if err != nil {
69+
logging.FromContext(ctx).Fatalf("Failed to get InClusterConfig: %v", err)
70+
}
71+
clientset, err := kubernetes.NewForConfig(config)
72+
if err != nil {
73+
logging.FromContext(ctx).Fatalf("Failed to get clientset: %v", err)
74+
}
75+
cm, err := clientset.CoreV1().ConfigMaps(*ns).Get(ctx, *cmname, metav1.GetOptions{})
76+
if err != nil {
77+
logging.FromContext(ctx).Fatalf("Failed to get the configmap %s/%s: %v", *ns, *cmname, err)
78+
}
79+
80+
if cm.Data == nil {
81+
cm.Data = make(map[string]string)
82+
}
83+
if treeID, ok := cm.Data[treeKey]; ok && !*force {
84+
logging.FromContext(ctx).Infof("Found existing TreeID: %s", treeID)
85+
return
86+
}
87+
88+
tree, err := createTree(ctx)
89+
if err != nil {
90+
logging.FromContext(ctx).Fatalf("Failed to create the trillian tree: %v", err)
91+
}
92+
cm.Data[treeKey] = fmt.Sprint(tree.TreeId)
93+
logging.FromContext(ctx).Infof("Created a new tree %d updating configmap %s/%s", tree.TreeId, *ns, *cmname)
94+
95+
_, err = clientset.CoreV1().ConfigMaps(*ns).Update(ctx, cm, metav1.UpdateOptions{})
96+
if err != nil {
97+
logging.FromContext(ctx).Fatalf("Failed to update the configmap: %v", err)
98+
}
99+
}
100+
101+
func createTree(ctx context.Context) (*trillian.Tree, error) {
102+
req, err := newRequest(ctx)
103+
if err != nil {
104+
return nil, err
105+
}
106+
107+
dialOpts, err := rpcflags.NewClientDialOptionsFromFlags()
108+
if err != nil {
109+
return nil, errors.Wrap(err, "failed to determine dial options")
110+
}
111+
112+
conn, err := grpc.Dial(*adminServerAddr, dialOpts...)
113+
if err != nil {
114+
return nil, errors.Wrap(err, "failed to dial")
115+
}
116+
defer conn.Close()
117+
118+
adminClient := trillian.NewTrillianAdminClient(conn)
119+
logClient := trillian.NewTrillianLogClient(conn)
120+
121+
return client.CreateAndInitTree(ctx, req, adminClient, logClient)
122+
}
123+
124+
func newRequest(ctx context.Context) (*trillian.CreateTreeRequest, error) {
125+
ts, ok := trillian.TreeState_value[*treeState]
126+
if !ok {
127+
return nil, fmt.Errorf("unknown TreeState: %v", *treeState)
128+
}
129+
130+
tt, ok := trillian.TreeType_value[*treeType]
131+
if !ok {
132+
return nil, fmt.Errorf("unknown TreeType: %v", *treeType)
133+
}
134+
135+
ctr := &trillian.CreateTreeRequest{Tree: &trillian.Tree{
136+
TreeState: trillian.TreeState(ts),
137+
TreeType: trillian.TreeType(tt),
138+
DisplayName: *displayName,
139+
Description: *description,
140+
MaxRootDuration: durationpb.New(*maxRootDuration),
141+
}}
142+
logging.FromContext(ctx).Infof("Creating Tree: %+v", ctr.Tree)
143+
144+
return ctr, nil
145+
}

‎examples/deployment/kubernetes/README-SCAFFOLDING.md‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
# Deploying onto Kubernetes
2+
3+
This document guides you through the process of spinning up an example Trillian
4+
deployment on Kubernetes cluster with [Knative](https://knative.dev/docs/)
5+
installed. It's suitable for GitHub action based e2e tests
6+
as well as local testing using something like kind cluster.
7+
8+
## Prerequisites
9+
10+
1. You should have this repo checked out :)
11+
1. A Kubernetes cluster with [Knative](https://knative.dev/docs/) installed. One
12+
example for installing local kind cluster with Knative is
13+
[here](https://github.com/sigstore/scaffolding/blob/main/getting-started.md#running-locally-on-kind)
14+
and ignoring the sigstore parts after installing the cluster.
15+
1. [ko](https://github.com/google/ko) installed.
16+
1. You have `kubectl` installed.
17+
18+
## Process
19+
20+
1. Create the scaffolding parts of the Trillian system.
21+
```shell
22+
kubectl apply -Rf ./examples/deployment/kubernetes/scaffolding
23+
```
24+
This spins up a namespace `trillian-system` where it deploys
25+
the following components:
26+
* log-signer
27+
* log-server
28+
* mysql server
29+
2. Let's make sure everything comes up ready:
30+
```shell
31+
kubectl wait -n trillian-system --for=condition=Ready --timeout=5m ksvc --all
32+
```
33+
34+
You should see something like this:
35+
```shell
36+
vaikas@villes-mbp scaffolding % kubectl wait -n trillian-system --for=condition=Ready --timeout=5m ksvc --all
37+
service.serving.knative.dev/log-server condition met
38+
service.serving.knative.dev/log-signer condition met
39+
```
40+
3. Then create a tree in the Trillian:
41+
```shell
42+
ko apply -BRf ./examples/deployment/kubernetes/createtree
43+
```
44+
45+
And make sure it completes.
46+
47+
```shell
48+
kubectl wait -n createtree --for=condition=Complete --timeout=5m jobs createtree
49+
```
50+
51+
4. Check out the tree:
52+
```shell
53+
kubectl -n createtree get cm trillian-tree -ojsonpath='{.data.treeID}'
54+
```
55+
56+
You should see something like this:
57+
```shell
58+
vaikas@villes-mbp scaffolding % kubectl -n createtree get cm trillian-tree -ojsonpath='{.data.treeID}'
59+
5213139395739357930%
60+
```
61+
62+
5. You can then use the TreeID for example to run CTLog on
63+
top of newly created Trillian.
64+
65+
6. TODO: Add examples for talking to Trillian for other things.

‎examples/deployment/kubernetes/createtree/100-namespace.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
kind: Namespace
2+
apiVersion: v1
3+
metadata:
4+
name: createtree

‎examples/deployment/kubernetes/createtree/101-binding.yaml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: Role
3+
metadata:
4+
namespace: createtree
5+
name: cm-operator
6+
rules:
7+
- apiGroups: [""] # "" indicates the core API group
8+
resources: ["configmaps"]
9+
resourceNames: ["trillian-tree"]
10+
verbs: ["get", "update"]
11+
---
12+
apiVersion: rbac.authorization.k8s.io/v1
13+
kind: RoleBinding
14+
metadata:
15+
name: role-cm-updater
16+
namespace: createtree
17+
roleRef:
18+
apiGroup: rbac.authorization.k8s.io
19+
kind: Role
20+
name: cm-operator
21+
subjects:
22+
- kind: ServiceAccount
23+
name: createtree
24+
namespace: createtree

‎examples/deployment/kubernetes/createtree/101-configmap.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: trillian-tree
5+
namespace: createtree
6+
data:
7+
__placeholder: |
8+
###################################################################
9+
# Just a placeholder so that reapplying this won't overwrite treeID
10+
# if it already exists. This caused grief, do not remove.
11+
###################################################################
12+

‎examples/deployment/kubernetes/createtree/101-service-account.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
apiVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: createtree
5+
namespace: createtree

‎examples/deployment/kubernetes/createtree/300-createtree.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
apiVersion: batch/v1
2+
kind: Job
3+
metadata:
4+
name: createtree
5+
namespace: createtree
6+
spec:
7+
template:
8+
spec:
9+
serviceAccountName: createtree
10+
restartPolicy: Never
11+
containers:
12+
- name: createtree
13+
image: ko://github.com/google/trillian/cmd/createtree-k8s
14+
args: [
15+
"--namespace=createtree",
16+
"--configmap=trillian-tree",
17+
"--display_name=ctlogtree"
18+
]

‎examples/deployment/kubernetes/scaffolding/log-server/100-namespace.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
kind: Namespace
2+
apiVersion: v1
3+
metadata:
4+
name: trillian-system

0 commit comments

Comments
 (0)