Skip to content

Commit 403732d

Browse files
author
Jeff McCormick
committed
fix load template logic
1 parent 062b312 commit 403732d

File tree

8 files changed

+91
-62
lines changed

8 files changed

+91
-62
lines changed

client/cmd/load.go

Lines changed: 67 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,19 @@
1717
package cmd
1818

1919
import (
20+
"bytes"
21+
"encoding/json"
2022
"fmt"
2123
log "github.com/Sirupsen/logrus"
2224
"github.com/crunchydata/postgres-operator/tpr"
2325
"github.com/spf13/cobra"
2426
"github.com/spf13/viper"
27+
"io/ioutil"
2528
"k8s.io/apimachinery/pkg/labels"
29+
"k8s.io/client-go/kubernetes"
30+
v1batch "k8s.io/client-go/pkg/apis/batch/v1"
31+
"os"
32+
"text/template"
2633
//"k8s.io/apimachinery/pkg/api/errors"
2734
//meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2835
//"time"
@@ -66,15 +73,27 @@ var loadCmd = &cobra.Command{
6673
},
6774
}
6875

76+
var CSVLOAD_TEMPLATE_PATH string
77+
78+
var JobTemplate *template.Template
79+
6980
func init() {
7081
RootCmd.AddCommand(loadCmd)
7182

7283
loadCmd.Flags().StringVarP(&Selector, "selector", "s", "", "The selector to use for cluster filtering ")
7384
loadCmd.Flags().StringVarP(&LoadConfig, "load-config", "l", "", "The load configuration to use that defines the load job")
74-
getLoadConfigFile()
85+
fmt.Println(" config is " + viper.GetString("PGO.CSVLOAD_TEMPLATE"))
7586
}
7687

7788
func createLoad(args []string) {
89+
CSVLOAD_TEMPLATE_PATH = viper.GetString("PGO.CSVLOAD_TEMPLATE")
90+
if CSVLOAD_TEMPLATE_PATH == "" {
91+
log.Error("PGO.CSVLOAD_TEMPLATE not defined in pgo config.")
92+
os.Exit(2)
93+
}
94+
getLoadConfigFile()
95+
fmt.Println("using csvload template from " + CSVLOAD_TEMPLATE_PATH)
96+
getJobTemplate()
7897
log.Debugf("createLoad called %v\n", args)
7998

8099
//var err error
@@ -117,17 +136,18 @@ func createLoad(args []string) {
117136
log.Debug("load called for " + arg)
118137

119138
fmt.Println("created load for " + arg)
139+
createJob(Clientset, arg, Namespace)
120140

121141
}
122142

123143
}
124144

125145
func getLoadConfigFile() {
126-
LoadConfigTemplate := LoadJobTemplateFields{}
146+
LoadConfigTemplate = LoadJobTemplateFields{}
127147
viper.SetConfigFile(LoadConfig)
128148
err := viper.ReadInConfig()
129149
if err == nil {
130-
log.Debugf("Using load config file:", viper.ConfigFileUsed())
150+
log.Debugf("Using load config file: %s\n", viper.ConfigFileUsed())
131151
} else {
132152
log.Debug("load config file not found")
133153
return
@@ -146,3 +166,47 @@ func getLoadConfigFile() {
146166
LoadConfigTemplate.SECURITY_CONTEXT = viper.GetString("SECURITY_CONTEXT")
147167

148168
}
169+
170+
func getJobTemplate() {
171+
var err error
172+
var buf []byte
173+
174+
buf, err = ioutil.ReadFile(CSVLOAD_TEMPLATE_PATH)
175+
if err != nil {
176+
log.Error("error loading csvload job template..." + err.Error())
177+
os.Exit(2)
178+
}
179+
JobTemplate = template.Must(template.New("csvload job template").Parse(string(buf)))
180+
181+
}
182+
183+
func createJob(clientset *kubernetes.Clientset, clusterName string, namespace string) {
184+
var err error
185+
186+
LoadConfigTemplate.Name = "csvload-" + clusterName
187+
LoadConfigTemplate.DB_HOST = clusterName
188+
189+
var doc2 bytes.Buffer
190+
err = JobTemplate.Execute(&doc2, LoadConfigTemplate)
191+
if err != nil {
192+
log.Error(err.Error())
193+
return
194+
}
195+
jobDocString := doc2.String()
196+
log.Debug(jobDocString)
197+
198+
newjob := v1batch.Job{}
199+
err = json.Unmarshal(doc2.Bytes(), &newjob)
200+
if err != nil {
201+
log.Error("error unmarshalling json into Job " + err.Error())
202+
return
203+
}
204+
205+
resultJob, err := Clientset.Batch().Jobs(Namespace).Create(&newjob)
206+
if err != nil {
207+
log.Error("error creating Job " + err.Error())
208+
return
209+
}
210+
log.Info("created load Job " + resultJob.Name)
211+
212+
}

conf/postgres-operator/csvload-job.json

Lines changed: 0 additions & 59 deletions
This file was deleted.

docs/config.asciidoc

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ BACKUP_STORAGE:
6666
SUPPLEMENTALGROUPS: 65534
6767
PGO:
6868
LSPVC_TEMPLATE: /home/youruserid/.pgo.lspvc-template.json
69+
CSVLOAD_TEMPLATE: /home/youruserid/.pgo.csvload-template.json
6970
CO_IMAGE_TAG: centos7-1.5.1
7071
DEBUG: false
7172
....
@@ -111,6 +112,7 @@ Values in the pgo configuration file have the following meaning:
111112
|BACKUP_STORAGE.FSGROUP | optional, if set, will cause a *SecurityContext* and *fsGroup* attributes to be added to generated Pod and Deployment definitions
112113
|BACKUP_STORAGE.SUPPLEMENTAL_GROUPS | optional, if set, will cause a SecurityContext to be added to generated Pod and Deployment definitions
113114
|PGO.LSPVC_TEMPLATE | the PVC lspvc template file used to view PVCs
115+
|PGO.CSVLOAD_TEMPLATE | the CSV load template file used for load jobs
114116
|PGO.CO_IMAGE_TAG | image tag to use for the PostgreSQL operator containers
115117
|PGO.DEBUG | set to true if you want to see debug messages from the pgo client
116118
|======================

examples/operator/csv-pvc.json

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"kind": "PersistentVolumeClaim",
3+
"apiVersion": "v1",
4+
"metadata": {
5+
"name": "csv-pvc"
6+
},
7+
"spec": {
8+
"accessModes": [
9+
"ReadWriteMany"
10+
],
11+
"resources": {
12+
"requests": {
13+
"storage": "100M"
14+
}
15+
}
16+
}
17+
}

examples/pgo.yaml.emptydir

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,5 +30,6 @@ REPLICA_STORAGE:
3030
SUPPLEMENTAL_GROUPS: 65534
3131
PGO:
3232
LSPVC_TEMPLATE: /home/jeffmc/.pgo.lspvc-template.json
33+
CSVLOAD_TEMPLATE: /home/jeffmc/.pgo.csvload-template.json
3334
CO_IMAGE_TAG: centos7-1.5.1
3435
DEBUG: false

examples/pgo.yaml.nfs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,5 +30,6 @@ REPLICA_STORAGE:
3030
SUPPLEMENTAL_GROUPS: 65534
3131
PGO:
3232
LSPVC_TEMPLATE: /home/jeffmc/.pgo.lspvc-template.json
33+
CSVLOAD_TEMPLATE: /home/jeffmc/.pgo.csvload-template.json
3334
CO_IMAGE_TAG: centos7-1.5.1
3435
DEBUG: false

examples/pgo.yaml.storageclass

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,5 +33,6 @@ REPLICA_STORAGE:
3333
FSGROUP: 26
3434
PGO:
3535
LSPVC_TEMPLATE: /home/jeffmc/.pgo.lspvc-template.json
36+
CSVLOAD_TEMPLATE: /home/jeffmc/.pgo.csvload-template.json
3637
CO_IMAGE_TAG: centos7-1.5.1
3738
DEBUG: false

examples/policy/xrayapp.sql

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
11
\c userdb;
22
create table xrayapp (id int, key string, value string);
33
create table a (id int);
4+
create table xraycsvtable (name string, state string, zip string);
5+

0 commit comments

Comments
 (0)