Skip to content

Commit 8177160

Browse files
author
jmccormick2001
committed
Merge branch 'develop'
2 parents ff792bf + 15c5041 commit 8177160

File tree

251 files changed

+3355
-1750
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

251 files changed

+3355
-1750
lines changed

‎ISSUE_TEMPLATE.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
- Operating System:
3737
- Container Image Tag:
3838
- Operator Version:
39+
- Storage (NFS, hostpath, storage class):
3940
- PostgreSQL Version:
4041
- Platform (Docker, Kubernetes, OpenShift):
4142
- Platform Version:

‎Makefile‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
RELTMPDIR=/tmp/release.$(CO_VERSION)
2+
HELMTMPDIR=/tmp/helm-release.$(CO_VERSION)
23
RELFILE=/tmp/postgres-operator.$(CO_VERSION).tar.gz
4+
HELMRELFILE=/tmp/postgres-operator-helm-chart.$(CO_VERSION).tar.gz
35

46
#======= Safety checks =======
57
check-go-vars:
@@ -47,7 +49,6 @@ installrbac:
4749
cd deploy && ./install-rbac.sh
4850
setup:
4951
./bin/get-deps.sh
50-
cd examples/backrest-config && ./create.sh
5152
setupnamespace:
5253
kubectl create -f ./examples/demo-namespace.json
5354
kubectl config set-context demo --cluster=kubernetes --namespace=demo --user=kubernetes-admin
@@ -70,6 +71,9 @@ pgo-backrest: check-go-vars
7071
pgo-backrest-image: check-go-vars pgo-backrest
7172
docker build -t pgo-backrest -f $(CO_BASEOS)/Dockerfile.pgo-backrest.$(CO_BASEOS) .
7273
docker tag pgo-backrest $(CO_IMAGE_PREFIX)/pgo-backrest:$(CO_IMAGE_TAG)
74+
foo: check-go-vars
75+
go install foo/foo.go
76+
mv $(GOBIN)/foo ./bin/foo/
7377
pgo: check-go-vars
7478
cd pgo && go install pgo.go
7579
clean: check-go-vars
@@ -124,11 +128,12 @@ pull:
124128
release: check-go-vars
125129
make macpgo
126130
make winpgo
127-
rm -rf $(RELTMPDIR) $(RELFILE)
128-
mkdir $(RELTMPDIR)
131+
rm -rf $(RELTMPDIR) $(RELFILE) $(HELMTMPDIR)
132+
mkdir $(RELTMPDIR) $(HELMTMPDIR)
129133
cp -r $(COROOT)/examples $(RELTMPDIR)
130134
cp -r $(COROOT)/deploy $(RELTMPDIR)
131135
cp -r $(COROOT)/conf $(RELTMPDIR)
136+
cp -r $(COROOT)/chart $(HELMTMPDIR)
132137
cp $(GOBIN)/pgo $(RELTMPDIR)
133138
cp $(GOBIN)/pgo-mac $(RELTMPDIR)
134139
cp $(GOBIN)/pgo.exe $(RELTMPDIR)
@@ -137,6 +142,7 @@ release: check-go-vars
137142
cp $(GOBIN)/expenv.exe $(RELTMPDIR)
138143
cp $(COROOT)/examples/pgo-bash-completion $(RELTMPDIR)
139144
tar czvf $(RELFILE) -C $(RELTMPDIR) .
145+
tar czvf $(HELMRELFILE) -C $(HELMTMPDIR) .
140146
default:
141147
all
142148

‎README.adoc‎

Lines changed: 54 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,14 @@
55
</p>
66
++++
77

8-
Latest Release: 3.3.0 {docdate}
8+
Latest Release: 3.4.0 {docdate}
99

1010
== General
1111

1212
The *postgres-operator* is a controller that runs within a Kubernetes cluster that provides a means to deploy and manage PostgreSQL clusters.
1313

14+
== Documentation
15+
1416
Please view the official Crunchy Data PostgreSQL Operator documentation link:https://crunchydata.github.io/postgres-operator/stable/[here]. If you are
1517
interested in contributing or making an update to the documentation, please view the link:https://crunchydata.github.io/postgres-operator/stable/contributing/[Contributing Guidelines].
1618

@@ -23,10 +25,60 @@ If you are looking for the latest documentation, please see the develop branch w
2325

2426
To build and deploy the Operator on your Kubernetes system, follow the instructions documented on the link:https://crunchydata.github.io/postgres-operator/stable/installation/[Installation] page.
2527

26-
If you're seeking to upgrade your existing Operator installation, please visit the link:https://crunchydata.github.io/postgres-operator/stable/installation/upgrading-the-operator/[Upgrading the Operator] page. Also read the Release Notes as to details on recent changes to the Operator.
28+
If you're seeking to upgrade your existing Operator installation, please visit the link:https://crunchydata.github.io/postgres-operator/stable/installation/upgrading-the-operator/[Upgrading the Operator] page. Also, read the Release Notes as to details on recent changes to the Operator.
29+
30+
== What is the Operator?
31+
32+
The *postgres-operator* is a controller that runs within a Kubernetes cluster that provides a means to deploy and manage PostgreSQL clusters.
33+
34+
Use the postgres-operator to -
35+
36+
* deploy PostgreSQL containers including streaming replication clusters
37+
* scale up PostgreSQL clusters with extra replicas
38+
* add pgpool and metrics sidecars to PostgreSQL clusters
39+
* apply SQL policies to PostgreSQL clusters
40+
* assign metadata tags to PostgreSQL clusters
41+
* maintain PostgreSQL users and passwords
42+
* perform minor and major upgrades to PostgreSQL clusters
43+
* load simple CSV and JSON files into PostgreSQL clusters
44+
* perform database backups
45+
46+
== Design
47+
48+
The *postgres-operator* design incorporates the following concepts -
49+
50+
* adds Custom Resource Definitions for PostgreSQL to Kubernetes
51+
* adds controller logic that watches events on PostgreSQL resources
52+
* provides a command line client (*pgo*) and REST API for interfacing with the postgres-operator
53+
* provides for very customized deployments including container resources, storage configurations, and PostgreSQL custom configurations
2754

2855
More design information is found on the link:https://crunchydata.github.io/postgres-operator/stable/how-it-works/[How It Works] page.
2956

57+
== Requirements
58+
59+
The postgres-operator runs on any Kubernetes and Openshift platform that supports
60+
Custom Resource Definitions.
61+
62+
The Operator project builds and operates with the following containers -
63+
64+
* link:https://hub.docker.com/r/crunchydata/pgo-lspvc/[PVC Listing Container]
65+
* link:https://hub.docker.com/r/crunchydata/pgo-rmdata/[Remove Data Container]
66+
* link:https://hub.docker.com/r/crunchydata/postgres-operator/[postgres-operator Container]
67+
* link:https://hub.docker.com/r/crunchydata/pgo-apiserver/[apiserver Container]
68+
* link:https://hub.docker.com/r/crunchydata/pgo-load/[file load Container]
69+
* link:https://hub.docker.com/r/crunchydata/pgo-backrest/[backrest interface Container]
70+
71+
This Operator is developed and tested on the following operating systems but is known to run on other operating systems -
72+
73+
* *CentOS 7*
74+
* *RHEL 7*
75+
76+
== Installation
77+
78+
To build and deploy the Operator on your Kubernetes system, follow the instructions documented on the link:https://crunchydata.github.io/postgres-operator/stable/installation/[Installation] page.
79+
80+
If you're seeking to upgrade your existing Operator installation, please visit the link:https://crunchydata.github.io/postgres-operator/installation/upgrading-the-operator/[Upgrading the Operator] page.
81+
3082
== Configuration
3183

3284
The operator is template-driven; this makes it simple to configure both the client and the operator. The configuration options are documented on the link:https://crunchydata.github.io/postgres-operator/stable/installation/configuration/[Configuration] page.

‎apis/cr/v1/backup.go‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -24,15 +24,15 @@ const PgbackupResourcePlural = "pgbackups"
2424

2525
// PgbackupSpec ...
2626
type PgbackupSpec struct {
27-
Name string `json:"name"`
28-
StorageSpec PgStorageSpec `json:"storagespec"`
29-
CCPImageTag string `json:"ccpimagetag"`
30-
BackupHost string `json:"backuphost"`
31-
BackupUser string `json:"backupuser"`
32-
BackupPass string `json:"backuppass"`
33-
BackupPort string `json:"backupport"`
34-
BackupStatus string `json:"backupstatus"`
35-
BackupPVC string `json:"backuppvc"`
27+
Name string `json:"name"`
28+
StorageSpec PgStorageSpec `json:"storagespec"`
29+
CCPImageTag string `json:"ccpimagetag"`
30+
BackupHost string `json:"backuphost"`
31+
BackupUserSecret string `json:"backupusersecret"`
32+
BackupPort string `json:"backupport"`
33+
BackupStatus string `json:"backupstatus"`
34+
BackupPVC string `json:"backuppvc"`
35+
BackupOpts string `json:"backupopts"`
3636
}
3737

3838
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

‎apis/cr/v1/cluster.go‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -43,23 +43,23 @@ type PgclusterSpec struct {
4343
ReplicaStorage PgStorageSpec `json:replicastorage`
4444
ContainerResources PgContainerResources `json:containerresources`
4545
PrimaryHost string `json:"primaryhost"`
46-
PrimaryPassword string `json:"primarypassword"`
47-
User string `json:"user"`
48-
Password string `json:"password"`
49-
Database string `json:"database"`
50-
RootPassword string `json:"rootpassword"`
51-
Replicas string `json:"replicas"`
52-
Strategy string `json:"strategy"`
53-
SecretFrom string `json:"secretfrom"`
54-
BackupPVCName string `json:"backuppvcname"`
55-
BackupPath string `json:"backuppath"`
56-
UserSecretName string `json:"usersecretname"`
57-
RootSecretName string `json:"rootsecretname"`
58-
PrimarySecretName string `json:"primarysecretname"`
59-
Status string `json:"status"`
60-
PswLastUpdate string `json:"pswlastupdate"`
61-
CustomConfig string `json:"customconfig"`
62-
UserLabels map[string]string `json:"userlabels"`
46+
//PrimaryPassword string `json:"primarypassword"`
47+
User string `json:"user"`
48+
//Password string `json:"password"`
49+
Database string `json:"database"`
50+
//RootPassword string `json:"rootpassword"`
51+
Replicas string `json:"replicas"`
52+
Strategy string `json:"strategy"`
53+
SecretFrom string `json:"secretfrom"`
54+
BackupPVCName string `json:"backuppvcname"`
55+
BackupPath string `json:"backuppath"`
56+
UserSecretName string `json:"usersecretname"`
57+
RootSecretName string `json:"rootsecretname"`
58+
PrimarySecretName string `json:"primarysecretname"`
59+
Status string `json:"status"`
60+
PswLastUpdate string `json:"pswlastupdate"`
61+
CustomConfig string `json:"customconfig"`
62+
UserLabels map[string]string `json:"userlabels"`
6363
}
6464

6565
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

‎apis/cr/v1/deepcopy.go‎

Lines changed: 26 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,15 @@ func (in *Pgbackup) DeepCopyInto(out *Pgbackup) {
88
out.TypeMeta = in.TypeMeta
99
out.ObjectMeta = in.ObjectMeta
1010
out.Spec = PgbackupSpec{
11-
Name: in.Spec.Name,
12-
StorageSpec: in.Spec.StorageSpec,
13-
CCPImageTag: in.Spec.CCPImageTag,
14-
BackupHost: in.Spec.BackupHost,
15-
BackupUser: in.Spec.BackupUser,
16-
BackupPass: in.Spec.BackupPass,
17-
BackupPort: in.Spec.BackupPort,
18-
BackupStatus: in.Spec.BackupStatus,
19-
BackupPVC: in.Spec.BackupPVC,
11+
Name: in.Spec.Name,
12+
StorageSpec: in.Spec.StorageSpec,
13+
CCPImageTag: in.Spec.CCPImageTag,
14+
BackupHost: in.Spec.BackupHost,
15+
BackupUserSecret: in.Spec.BackupUserSecret,
16+
BackupPort: in.Spec.BackupPort,
17+
BackupOpts: in.Spec.BackupOpts,
18+
BackupStatus: in.Spec.BackupStatus,
19+
BackupPVC: in.Spec.BackupPVC,
2020
}
2121
out.Status = in.Status
2222
}
@@ -150,23 +150,23 @@ func (in *Pgcluster) DeepCopyInto(out *Pgcluster) {
150150
ReplicaStorage: in.Spec.ReplicaStorage,
151151
ContainerResources: in.Spec.ContainerResources,
152152
PrimaryHost: in.Spec.PrimaryHost,
153-
PrimaryPassword: in.Spec.PrimaryPassword,
154-
User: in.Spec.User,
155-
Password: in.Spec.Password,
156-
Database: in.Spec.Database,
157-
RootPassword: in.Spec.RootPassword,
158-
Replicas: in.Spec.Replicas,
159-
Strategy: in.Spec.Strategy,
160-
SecretFrom: in.Spec.SecretFrom,
161-
BackupPVCName: in.Spec.BackupPVCName,
162-
BackupPath: in.Spec.BackupPath,
163-
UserSecretName: in.Spec.UserSecretName,
164-
RootSecretName: in.Spec.RootSecretName,
165-
PrimarySecretName: in.Spec.PrimarySecretName,
166-
Status: in.Spec.Status,
167-
PswLastUpdate: in.Spec.PswLastUpdate,
168-
CustomConfig: in.Spec.CustomConfig,
169-
UserLabels: in.Spec.UserLabels,
153+
//PrimaryPassword: in.Spec.PrimaryPassword,
154+
User: in.Spec.User,
155+
//Password: in.Spec.Password,
156+
Database: in.Spec.Database,
157+
//RootPassword: in.Spec.RootPassword,
158+
Replicas: in.Spec.Replicas,
159+
Strategy: in.Spec.Strategy,
160+
SecretFrom: in.Spec.SecretFrom,
161+
BackupPVCName: in.Spec.BackupPVCName,
162+
BackupPath: in.Spec.BackupPath,
163+
UserSecretName: in.Spec.UserSecretName,
164+
RootSecretName: in.Spec.RootSecretName,
165+
PrimarySecretName: in.Spec.PrimarySecretName,
166+
Status: in.Spec.Status,
167+
PswLastUpdate: in.Spec.PswLastUpdate,
168+
CustomConfig: in.Spec.CustomConfig,
169+
UserLabels: in.Spec.UserLabels,
170170
}
171171
}
172172

‎apis/cr/v1/task.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,12 @@ const PgtaskFailover = "failover"
3434
const PgtaskAutoFailover = "autofailover"
3535
const PgtaskAddPolicies = "addpolicies"
3636

37+
const PgtaskWorkflow = "workflow"
38+
const PgtaskWorkflowCreateClusterType = "createcluster"
39+
const PgtaskWorkflowSubmittedStatus = "submitted"
40+
const PgtaskWorkflowCompletedStatus = "completed"
41+
const PgtaskWorkflowID = "workflowid"
42+
3743
const PgtaskBackrest = "backrest"
3844
const PgtaskBackrestBackup = "backup"
3945
const PgtaskBackrestInfo = "info"

‎apiserver.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,11 +44,12 @@ import (
4444
"github.com/crunchydata/postgres-operator/apiserver/upgradeservice"
4545
"github.com/crunchydata/postgres-operator/apiserver/userservice"
4646
"github.com/crunchydata/postgres-operator/apiserver/versionservice"
47+
"github.com/crunchydata/postgres-operator/apiserver/workflowservice"
4748
"github.com/gorilla/mux"
4849
)
4950

50-
const serverCert = "/config/server.crt"
51-
const serverKey = "/config/server.key"
51+
const serverCert = "/pgo-auth-secret/server.crt"
52+
const serverKey = "/pgo-auth-secret/server.key"
5253

5354
func main() {
5455

@@ -85,6 +86,7 @@ func main() {
8586
r.HandleFunc("/policies/{name}", policyservice.ShowPolicyHandler).Methods("GET")
8687
//here
8788
r.HandleFunc("/policiesdelete/{name}", policyservice.DeletePolicyHandler).Methods("GET")
89+
r.HandleFunc("/workflow/{id}", workflowservice.ShowWorkflowHandler).Methods("GET")
8890
r.HandleFunc("/pvc/{pvcname}", pvcservice.ShowPVCHandler).Methods("GET")
8991
r.HandleFunc("/policies/apply", policyservice.ApplyPolicyHandler).Methods("POST")
9092
r.HandleFunc("/ingest", ingestservice.CreateIngestHandler).Methods("POST")

‎apiserver/backrestservice/backrestimpl.go‎

Lines changed: 35 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ import (
2929
)
3030

3131
const backrestCommand = "pgbackrest"
32-
const backrestStanza = "--stanza=db"
3332
const backrestInfoCommand = "info"
3433
const containername = "database"
3534

@@ -134,7 +133,7 @@ func CreateBackup(request *msgs.CreateBackrestBackupRequest) msgs.CreateBackrest
134133
}
135134

136135
//get pod name from cluster
137-
var podname string
136+
var podname, deployName string
138137
podname, err = getPrimaryPodName(&cluster)
139138

140139
if err != nil {
@@ -144,7 +143,16 @@ func CreateBackup(request *msgs.CreateBackrestBackupRequest) msgs.CreateBackrest
144143
return resp
145144
}
146145

147-
err = kubeapi.Createpgtask(apiserver.RESTClient, getBackupParams(clusterName, taskName, crv1.PgtaskBackrestBackup, podname, "database", request.BackupOpts), apiserver.Namespace)
146+
//get deployment name for this cluster
147+
deployName, err = getDeployName(&cluster)
148+
if err != nil {
149+
log.Error(err)
150+
resp.Status.Code = msgs.Error
151+
resp.Status.Msg = err.Error()
152+
return resp
153+
}
154+
155+
err = kubeapi.Createpgtask(apiserver.RESTClient, getBackupParams(deployName, clusterName, taskName, crv1.PgtaskBackrestBackup, podname, "database", request.BackupOpts), apiserver.Namespace)
148156
if err != nil {
149157
resp.Status.Code = msgs.Error
150158
resp.Status.Msg = err.Error()
@@ -157,7 +165,7 @@ func CreateBackup(request *msgs.CreateBackrestBackupRequest) msgs.CreateBackrest
157165
return resp
158166
}
159167

160-
func getBackupParams(clusterName, taskName, action, podName, containerName, backupOpts string) *crv1.Pgtask {
168+
func getBackupParams(deployName, clusterName, taskName, action, podName, containerName, backupOpts string) *crv1.Pgtask {
161169
var newInstance *crv1.Pgtask
162170

163171
spec := crv1.PgtaskSpec{}
@@ -183,6 +191,26 @@ func removeBackupJob(clusterName string) {
183191

184192
}
185193

194+
func getDeployName(cluster *crv1.Pgcluster) (string, error) {
195+
var depName string
196+
197+
selector := util.LABEL_PGPOOL + "!=true," + util.LABEL_PG_CLUSTER + "=" + cluster.Spec.Name + "," + util.LABEL_PRIMARY + "=true"
198+
199+
deps, err := kubeapi.GetDeployments(apiserver.Clientset, selector, apiserver.Namespace)
200+
if err != nil {
201+
return depName, err
202+
}
203+
204+
if len(deps.Items) != 1 {
205+
return depName, errors.New("error: deployment count is wrong for backrest backup " + cluster.Spec.Name)
206+
}
207+
for _, d := range deps.Items {
208+
return d.Name, err
209+
}
210+
211+
return depName, errors.New("unknown error in backrest backup")
212+
}
213+
186214
func getPrimaryPodName(cluster *crv1.Pgcluster) (string, error) {
187215
var podname string
188216

@@ -291,7 +319,6 @@ func getInfo(clusterName, podname string) (string, error) {
291319
log.Info("backrest info command requested")
292320
//pgbackrest --stanza=db info
293321
cmd = append(cmd, backrestCommand)
294-
cmd = append(cmd, backrestStanza)
295322
cmd = append(cmd, backrestInfoCommand)
296323

297324
log.Infof("command is %v ", cmd)
@@ -382,6 +409,9 @@ func getRestoreParams(request *msgs.RestoreRequest) *crv1.Pgtask {
382409
spec.Parameters[util.LABEL_BACKREST_RESTORE_FROM_CLUSTER] = request.FromCluster
383410
spec.Parameters[util.LABEL_BACKREST_RESTORE_TO_PVC] = request.ToPVC
384411
spec.Parameters[util.LABEL_BACKREST_RESTORE_OPTS] = request.RestoreOpts
412+
spec.Parameters[util.LABEL_PGBACKREST_STANZA] = "db"
413+
spec.Parameters[util.LABEL_PGBACKREST_DB_PATH] = "/pgdata/" + request.ToPVC
414+
spec.Parameters[util.LABEL_PGBACKREST_REPO_PATH] = "/backrestrepo/" + request.FromCluster + "-backups"
385415

386416
newInstance = &crv1.Pgtask{
387417
ObjectMeta: meta_v1.ObjectMeta{

0 commit comments

Comments
 (0)