Skip to content

Commit 0d6a4d4

Browse files
author
Jeff Peeler
committed
feat(filemonitor): add new filemonitor package
Watcher is a generic (also cross platform) file watching mechanism for receiving file system events in real time. A simple API has been written to watch a list of paths and to run a user defined callback for processing of the events. Using the above, a keystore has been made such that when a target directory containing certificates are updated the contents are made available via a callback. This callback is compatible with passing to an http server tls config, which allows for zero downtime certificate swapping.
1 parent ef79cec commit 0d6a4d4

File tree

12 files changed

+513
-0
lines changed

12 files changed

+513
-0
lines changed

‎pkg/lib/filemonitor/cert_updater.go

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
package filemonitor
2+
3+
import (
4+
"context"
5+
"crypto/tls"
6+
"crypto/x509"
7+
"fmt"
8+
"path/filepath"
9+
"sync"
10+
11+
"github.com/fsnotify/fsnotify"
12+
"github.com/sirupsen/logrus"
13+
)
14+
15+
type keystore struct {
16+
mutex sync.RWMutex
17+
cert *tls.Certificate
18+
tlsCrtPath string
19+
tlsKeyPath string
20+
}
21+
22+
type getCertFn = func(*tls.ClientHelloInfo) (*tls.Certificate, error)
23+
24+
// NewKeystore returns a store for storing the certificate data and the ability to retrieve it safely
25+
func NewKeystore(tlsCrt, tlsKey string) *keystore {
26+
cert, err := tls.LoadX509KeyPair(tlsCrt, tlsKey)
27+
if err != nil {
28+
panic(err)
29+
}
30+
return &keystore{
31+
mutex: sync.RWMutex{},
32+
cert: &cert,
33+
tlsCrtPath: tlsCrt,
34+
tlsKeyPath: tlsKey,
35+
}
36+
}
37+
38+
// HandleFilesystemUpdate is intended to be used as the OnUpdateFn for a watcher
39+
// and expects the certificate files to be in the same directory.
40+
func (k *keystore) HandleFilesystemUpdate(logger *logrus.Logger, event fsnotify.Event) {
41+
switch op := event.Op; op {
42+
case fsnotify.Create:
43+
logger.Debugf("got fs event for %v", event.Name)
44+
45+
if err := k.storeCertificate(k.tlsCrtPath, k.tlsKeyPath); err != nil {
46+
// this can happen if both certificates aren't updated at the same
47+
// time, but it's okay as replacement only occurs with a valid key pair
48+
logger.Debugf("certificates not in sync: %v", err)
49+
} else {
50+
info, err := x509.ParseCertificate(k.cert.Certificate[0])
51+
if err != nil {
52+
logger.Debugf("certificates refreshed, but parsing returned error: %v", err)
53+
} else {
54+
logger.Debugf("certificates refreshed: Subject=%v NotBefore=%v NotAfter=%v", info.Subject, info.NotBefore, info.NotAfter)
55+
}
56+
}
57+
}
58+
}
59+
60+
func (k *keystore) storeCertificate(tlsCrt, tlsKey string) error {
61+
cert, err := tls.LoadX509KeyPair(tlsCrt, tlsKey)
62+
if err == nil {
63+
k.mutex.Lock()
64+
defer k.mutex.Unlock()
65+
k.cert = &cert
66+
}
67+
return err
68+
}
69+
70+
func (k *keystore) GetCertificate(h *tls.ClientHelloInfo) (*tls.Certificate, error) {
71+
k.mutex.RLock()
72+
defer k.mutex.RUnlock()
73+
return k.cert, nil
74+
}
75+
76+
// OLMGetCertRotationFn is a convenience function for OLM use only, but serves as an example for monitoring file system events
77+
func OLMGetCertRotationFn(logger *logrus.Logger, tlsCertPath, tlsKeyPath string) (getCertFn, error) {
78+
if filepath.Dir(tlsCertPath) != filepath.Dir(tlsKeyPath) {
79+
return nil, fmt.Errorf("certificates expected to be in same directory %v vs %v", tlsCertPath, tlsKeyPath)
80+
}
81+
82+
keystore := NewKeystore(tlsCertPath, tlsKeyPath)
83+
watcher, err := NewWatch(logger, []string{filepath.Dir(tlsCertPath)}, keystore.HandleFilesystemUpdate)
84+
if err != nil {
85+
return nil, err
86+
}
87+
watcher.Run(context.Background())
88+
89+
return keystore.GetCertificate, nil
90+
}

‎pkg/lib/filemonitor/cert_updater_test.go

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
package filemonitor
2+
3+
import (
4+
"crypto/tls"
5+
"crypto/x509"
6+
"fmt"
7+
"html"
8+
"io/ioutil"
9+
"net"
10+
"net/http"
11+
"os"
12+
"path/filepath"
13+
"strconv"
14+
"testing"
15+
"time"
16+
17+
"k8s.io/apimachinery/pkg/util/wait"
18+
19+
"github.com/sirupsen/logrus"
20+
21+
"github.com/stretchr/testify/assert"
22+
"github.com/stretchr/testify/require"
23+
)
24+
25+
func TestOLMGetCertRotationFn(t *testing.T) {
26+
logger := logrus.New()
27+
logger.SetLevel(logrus.DebugLevel)
28+
logger.SetFormatter(&logrus.TextFormatter{
29+
TimestampFormat: time.RFC3339Nano,
30+
})
31+
32+
testData := "testdata"
33+
monitorDir := "monitor"
34+
caCrt := filepath.Join(testData, "ca.crt")
35+
oldCrt := filepath.Join(testData, "server-old.crt")
36+
oldKey := filepath.Join(testData, "server-old.key")
37+
newCrt := filepath.Join(testData, "server-new.crt")
38+
newKey := filepath.Join(testData, "server-new.key")
39+
loadCrt := filepath.Join(monitorDir, "loaded.crt")
40+
loadKey := filepath.Join(monitorDir, "loaded.key")
41+
42+
// these values must match values specified in the testdata generation script
43+
expectedOldCN := "CN=127.0.0.1,OU=OpenShift,O=Red Hat,L=Columbia,ST=SC,C=US"
44+
expectedNewCN := "CN=127.0.0.1,OU=OpenShift,O=Red Hat,L=New York City,ST=NY,C=US"
45+
46+
// the directory is expected to contain exactly one keypair, so create an empty directory to swap the keys in
47+
err := os.RemoveAll(monitorDir) // this is for test development, shouldn't ever exist beforehand otherwise
48+
require.NoError(t, err)
49+
err = os.Mkdir(monitorDir, 0777)
50+
require.NoError(t, err)
51+
52+
// symlink old files to loading files
53+
err = os.Symlink(filepath.Join("..", oldCrt), loadCrt)
54+
require.NoError(t, err)
55+
err = os.Symlink(filepath.Join("..", oldKey), loadKey)
56+
require.NoError(t, err)
57+
58+
tlsGetCertFn, err := OLMGetCertRotationFn(logger, loadCrt, loadKey)
59+
require.NoError(t, err)
60+
61+
// find a free port to listen on and start server
62+
listener, err := net.Listen("tcp", ":0")
63+
require.NoError(t, err)
64+
freePort := listener.Addr().(*net.TCPAddr).Port
65+
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
66+
fmt.Fprintf(w, "Path: %q", html.EscapeString(r.URL.Path))
67+
})
68+
httpsServer := &http.Server{
69+
Addr: ":" + strconv.Itoa(freePort),
70+
TLSConfig: &tls.Config{
71+
GetCertificate: tlsGetCertFn,
72+
},
73+
}
74+
go func() {
75+
if err := httpsServer.ServeTLS(listener, "", ""); err != nil {
76+
panic(err)
77+
}
78+
}()
79+
80+
caCert, err := ioutil.ReadFile(caCrt)
81+
require.NoError(t, err)
82+
caCertPool := x509.NewCertPool()
83+
caCertPool.AppendCertsFromPEM(caCert)
84+
85+
client := &http.Client{
86+
Transport: &http.Transport{
87+
TLSClientConfig: &tls.Config{
88+
RootCAs: caCertPool,
89+
},
90+
},
91+
}
92+
93+
resp, err := client.Get(fmt.Sprintf("https://localhost:%v", freePort))
94+
require.NoError(t, err)
95+
assert.Equal(t, resp.StatusCode, http.StatusOK)
96+
assert.Equal(t, expectedOldCN, resp.TLS.PeerCertificates[0].Subject.String())
97+
resp.Body.Close()
98+
client.CloseIdleConnections()
99+
100+
// atomically switch out the symlink so the file contents are always seen in a consistent state
101+
// (the same idea is used in the atomic writer in kubernetes)
102+
atomicCrt := loadCrt + ".atomic-op"
103+
atomicKey := loadKey + ".atomic-op"
104+
err = os.Symlink(filepath.Join("..", newCrt), atomicCrt)
105+
require.NoError(t, err)
106+
err = os.Symlink(filepath.Join("..", newKey), atomicKey)
107+
require.NoError(t, err)
108+
109+
err = os.Rename(atomicCrt, loadCrt)
110+
require.NoError(t, err)
111+
err = os.Rename(atomicKey, loadKey)
112+
require.NoError(t, err)
113+
114+
// sometimes the the filesystem operations need time to catch up so the server cert is updated
115+
err = wait.PollImmediate(500*time.Millisecond, 10*time.Second, func() (bool, error) {
116+
currentCert, _ := tlsGetCertFn(nil)
117+
info, err := x509.ParseCertificate(currentCert.Certificate[0])
118+
if err != nil {
119+
return false, err
120+
}
121+
if info.Subject.String() == expectedNewCN {
122+
return true, nil
123+
}
124+
125+
return false, nil
126+
})
127+
require.NoError(t, err)
128+
129+
resp, err = client.Get(fmt.Sprintf("https://localhost:%v", freePort))
130+
require.NoError(t, err)
131+
assert.Equal(t, resp.StatusCode, http.StatusOK)
132+
assert.Equal(t, expectedNewCN, resp.TLS.PeerCertificates[0].Subject.String())
133+
134+
os.RemoveAll(monitorDir)
135+
}

‎pkg/lib/filemonitor/testdata/ca.crt

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
-----BEGIN CERTIFICATE-----
2+
MIIDCTCCAfGgAwIBAgIUZSqEGDsDVte9dO9YVGNd3n9/EakwDQYJKoZIhvcNAQEL
3+
BQAwFDESMBAGA1UEAwwJMTI3LjAuMC4xMB4XDTE5MTEyMzAyMDM1OFoXDTQ3MDQx
4+
MDAyMDM1OFowFDESMBAGA1UEAwwJMTI3LjAuMC4xMIIBIjANBgkqhkiG9w0BAQEF
5+
AAOCAQ8AMIIBCgKCAQEA4Gwb6+HM2IJn/ZZIERVA/ngKQzk+Lk+36Yrn1ka8HOum
6+
vmWhozle3FwczCOQks5Jc2v5W6ulZHg0uQtNutw3lWkinVqGZzI34E0bxKchPniy
7+
egGnW3n9/dgIVd9ooPnOLJJXEwV4ZVIhubfB/EHib9rha2nIKVyYxihhrkCMU8aq
8+
qhYqOzJYk9eWzwB+mXAxW5AMmUAmVF0J9JvpG/FHliG+WgJNrxunuGuIa4XLw50H
9+
V5bJ5pXUdX/L0EPTQ1KWpmjByFD/dcOmFGc+pdw+x84CYUAH2DkxewUHAe0e6hlO
10+
RY3tuOXEigtfU6F0jUxD2RjC2pBp1WoH+UyyUiRQTQIDAQABo1MwUTAdBgNVHQ4E
11+
FgQU0jPw9F7SBqdbDbws33KFGPqKqb8wHwYDVR0jBBgwFoAU0jPw9F7SBqdbDbws
12+
33KFGPqKqb8wDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAjDnW
13+
CiQ8X+Ncm09bF6SXjtTzwbSduJ7VEkVjVm+nW0/Rh56Lgw0Z/ceWy/GyqYzuhsUz
14+
6uQOrS3drlg+p2fn4ZqtKWmDByYaUQS709I/qDkBE9mXxjojOhnQEqgjdQ5yoIP7
15+
mXeJQlFlrSaf4yEW264XQjbasKGQi2QBejhHK+aHw6PsszaNxqeqvTr0K95OTPPv
16+
vl2L0A4grMKtXnws8LL6zazz9eqib0K9iyHOLxvtWnPjc1y3XmFPr8mjWmbc9pl3
17+
5UGohvZy/wBjPS0YPLMDoGGu0jajvbNqcoEMFJ6Zo6oPGJIpwpqCjgLAgwMpx9oL
18+
EhMnEidqcop3+Z28kA==
19+
-----END CERTIFICATE-----

‎pkg/lib/filemonitor/testdata/cert-config/Makefile

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
DESTINATION:=../
2+
3+
all:
4+
./gen-certs.sh
5+
6+
clean:
7+
rm -f *.csr *.crt *.key *.srl
8+
9+
copy:
10+
cp ca.crt $(DESTINATION)
11+
cp server-old.crt $(DESTINATION)
12+
cp server-old.key $(DESTINATION)
13+
cp server-new.crt $(DESTINATION)
14+
cp server-new.key $(DESTINATION)

‎pkg/lib/filemonitor/testdata/cert-config/csr-new.conf

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
[ req ]
2+
default_bits = 2048
3+
prompt = no
4+
default_md = sha256
5+
req_extensions = req_ext
6+
distinguished_name = dn
7+
8+
[ dn ]
9+
C = US
10+
ST = NY
11+
L = New York City
12+
O = Red Hat
13+
OU = OpenShift
14+
CN = 127.0.0.1
15+
16+
[ req_ext ]
17+
subjectAltName = @alt_names
18+
19+
[ alt_names ]
20+
DNS.1 = localhost
21+
IP.1 = 127.0.0.1
22+
23+
[ v3_ext ]
24+
authorityKeyIdentifier=keyid,issuer:always
25+
basicConstraints=CA:FALSE
26+
keyUsage=keyEncipherment,dataEncipherment
27+
extendedKeyUsage=serverAuth,clientAuth
28+
subjectAltName=@alt_names

‎pkg/lib/filemonitor/testdata/cert-config/csr-old.conf

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
[ req ]
2+
default_bits = 2048
3+
prompt = no
4+
default_md = sha256
5+
req_extensions = req_ext
6+
distinguished_name = dn
7+
8+
[ dn ]
9+
C = US
10+
ST = SC
11+
L = Columbia
12+
O = Red Hat
13+
OU = OpenShift
14+
CN = 127.0.0.1
15+
16+
[ req_ext ]
17+
subjectAltName = @alt_names
18+
19+
[ alt_names ]
20+
DNS.1 = localhost
21+
IP.1 = 127.0.0.1
22+
23+
[ v3_ext ]
24+
authorityKeyIdentifier=keyid,issuer:always
25+
basicConstraints=CA:FALSE
26+
keyUsage=keyEncipherment,dataEncipherment
27+
extendedKeyUsage=serverAuth,clientAuth
28+
subjectAltName=@alt_names

‎pkg/lib/filemonitor/testdata/cert-config/gen-certs.sh

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
#!/bin/bash
2+
# Based off:
3+
# https://kubernetes.io/docs/concepts/cluster-administration/certificates/
4+
#
5+
# This scripts generates self-signed certificate keypairs, with the only
6+
# difference being that the subjects are different (set in $CSR) to easily allow
7+
# detecting which are in use.
8+
9+
function set_variables {
10+
MASTER_IP="127.0.0.1"
11+
CA_CRT=ca.crt
12+
CA_KEY=ca.key
13+
CSR=csr-$SUFFIX.conf
14+
SERVER_CSR=server-$SUFFIX.csr
15+
SERVER_CRT=server-$SUFFIX.crt
16+
SERVER_KEY=server-$SUFFIX.key
17+
}
18+
19+
function generate_ca {
20+
openssl genrsa -out $CA_KEY 2048
21+
openssl req -x509 -new -nodes -key $CA_KEY -subj "/CN=${MASTER_IP}" -days 10000 -out $CA_CRT
22+
}
23+
24+
function generate_certs {
25+
echo "Generating certs for $SUFFIX"
26+
openssl genrsa -out "$SERVER_KEY" 2048
27+
openssl req -new -key "$SERVER_KEY" -out "$SERVER_CSR" -config "$CSR"
28+
openssl x509 -req -in "$SERVER_CSR" -CA $CA_CRT -CAkey "$CA_KEY" -CAcreateserial -out "$SERVER_CRT" -days 10000 -extensions v3_ext -extfile "$CSR"
29+
#openssl x509 -noout -text -in "$SERVER_CRT"
30+
echo "---"
31+
}
32+
33+
34+
SUFFIX=old
35+
set_variables
36+
generate_ca # do this only once
37+
generate_certs
38+
39+
SUFFIX=new
40+
set_variables
41+
generate_certs

‎pkg/lib/filemonitor/testdata/server-new.crt

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
-----BEGIN CERTIFICATE-----
2+
MIIDtjCCAp6gAwIBAgIUTxuDVC0t2atYlgVW5iZ7Fz/9DMQwDQYJKoZIhvcNAQEL
3+
BQAwFDESMBAGA1UEAwwJMTI3LjAuMC4xMB4XDTE5MTEyMzAyMDM1OFoXDTQ3MDQx
4+
MDAyMDM1OFowbDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAk5ZMRYwFAYDVQQHDA1O
5+
ZXcgWW9yayBDaXR5MRAwDgYDVQQKDAdSZWQgSGF0MRIwEAYDVQQLDAlPcGVuU2hp
6+
ZnQxEjAQBgNVBAMMCTEyNy4wLjAuMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
7+
AQoCggEBAL2SdYBu/Gee0CbxIAHAHaBRpOA6ZjzbBgiWbLDfvexqoNU3aR5ryjE/
8+
8QOdHxYQMon8ymDut6I/SF/XS+fke0WPtPmP1DyX2wAhzJLjpIR9GX1AVayHbKMB
9+
/kxbyI8ow0qmCQhtnCvBb4JWBDWYwn1t9qpHpE3E8kbM2sh4jI/03nxZkLv63qhE
10+
kLxXd1dnOJOGOsQEHULWyMSo0sRyEOvr5gNz4YlhxAITvdQAhjRU0HTASKTvEez7
11+
Ksm18S25tc/nE61Ql1QlXAHZF9/nIH9ZOKtHvZi1vRzwUlfmAMd5PeShfSRxGPYt
12+
nrOZo3i4fuppwY17Ek2+4OG03oomlZMCAwEAAaOBpzCBpDBPBgNVHSMESDBGgBTS
13+
M/D0XtIGp1sNvCzfcoUY+oqpv6EYpBYwFDESMBAGA1UEAwwJMTI3LjAuMC4xghRl
14+
KoQYOwNW171071hUY13ef38RqTAJBgNVHRMEAjAAMAsGA1UdDwQEAwIEMDAdBgNV
15+
HSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwGgYDVR0RBBMwEYIJbG9jYWxob3N0
16+
hwR/AAABMA0GCSqGSIb3DQEBCwUAA4IBAQA14DWLtAPYC2jYAwrQTbV7NeI9hkKX
17+
n4Zhf2R/aN30gXKD4yKfi1QOxoOdywaJD4659Q5trJkRB1mOE2pKsuL+PsLd8Z4h
18+
7K7D2avsTf06Dwl4LBjhCuWNKqIJFATRYX5BkbBvm2N3R/Z8tu31IAzyCZGItTVw
19+
yLzttjiZ43SzoVmHGbIkaqYzmUkhgknZsrQAF09e7JxfsOmAk2T6ykad7lnJq12C
20+
mrsSwH193Yx/8YdWokxZxZeGpt5lyeZZJU05DU5LXydpQh7t/XdfE9Y6zpqEaH1R
21+
Ra7xE8IwC9t1c+Z1tcnXKxEgJ/2e1Zm7nqHQC+D5Y3hEngAJYRi8OM+Q
22+
-----END CERTIFICATE-----

0 commit comments

Comments
 (0)