fix load template logic

Author: Jeff McCormick

client/cmd/load.go

@@ -17,12 +17,19 @@
 package cmd
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
 	log "github.com/Sirupsen/logrus"
 	"github.com/crunchydata/postgres-operator/tpr"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	"io/ioutil"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/kubernetes"
+	v1batch "k8s.io/client-go/pkg/apis/batch/v1"
+	"os"
+	"text/template"
 	//"k8s.io/apimachinery/pkg/api/errors"
 	//meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	//"time"
@@ -66,15 +73,27 @@ var loadCmd = &cobra.Command{
 	},
 }
 
+var CSVLOAD_TEMPLATE_PATH string
+
+var JobTemplate *template.Template
+
 func init() {
 	RootCmd.AddCommand(loadCmd)
 
 	loadCmd.Flags().StringVarP(&Selector, "selector", "s", "", "The selector to use for cluster filtering ")
 	loadCmd.Flags().StringVarP(&LoadConfig, "load-config", "l", "", "The load configuration to use that defines the load job")
-	getLoadConfigFile()
+	fmt.Println(" config is " + viper.GetString("PGO.CSVLOAD_TEMPLATE"))
 }
 
 func createLoad(args []string) {
+	CSVLOAD_TEMPLATE_PATH = viper.GetString("PGO.CSVLOAD_TEMPLATE")
+	if CSVLOAD_TEMPLATE_PATH == "" {
+		log.Error("PGO.CSVLOAD_TEMPLATE not defined in pgo config.")
+		os.Exit(2)
+	}
+	getLoadConfigFile()
+	fmt.Println("using csvload template from " + CSVLOAD_TEMPLATE_PATH)
+	getJobTemplate()
 	log.Debugf("createLoad called %v\n", args)
 
 	//var err error
@@ -117,17 +136,18 @@ func createLoad(args []string) {
 		log.Debug("load called for " + arg)
 
 		fmt.Println("created load for " + arg)
+		createJob(Clientset, arg, Namespace)
 
 	}
 
 }
 
 func getLoadConfigFile() {
-	LoadConfigTemplate := LoadJobTemplateFields{}
+	LoadConfigTemplate = LoadJobTemplateFields{}
 	viper.SetConfigFile(LoadConfig)
 	err := viper.ReadInConfig()
 	if err == nil {
-		log.Debugf("Using load config file:", viper.ConfigFileUsed())
+		log.Debugf("Using load config file: %s\n", viper.ConfigFileUsed())
 	} else {
 		log.Debug("load config file not found")
 		return
@@ -146,3 +166,47 @@ func getLoadConfigFile() {
 	LoadConfigTemplate.SECURITY_CONTEXT = viper.GetString("SECURITY_CONTEXT")
 
 }
+
+func getJobTemplate() {
+	var err error
+	var buf []byte
+
+	buf, err = ioutil.ReadFile(CSVLOAD_TEMPLATE_PATH)
+	if err != nil {
+		log.Error("error loading csvload job template..." + err.Error())
+		os.Exit(2)
+	}
+	JobTemplate = template.Must(template.New("csvload job template").Parse(string(buf)))
+
+}
+
+func createJob(clientset *kubernetes.Clientset, clusterName string, namespace string) {
+	var err error
+
+	LoadConfigTemplate.Name = "csvload-" + clusterName
+	LoadConfigTemplate.DB_HOST = clusterName
+
+	var doc2 bytes.Buffer
+	err = JobTemplate.Execute(&doc2, LoadConfigTemplate)
+	if err != nil {
+		log.Error(err.Error())
+		return
+	}
+	jobDocString := doc2.String()
+	log.Debug(jobDocString)
+
+	newjob := v1batch.Job{}
+	err = json.Unmarshal(doc2.Bytes(), &newjob)
+	if err != nil {
+		log.Error("error unmarshalling json into Job " + err.Error())
+		return
+	}
+
+	resultJob, err := Clientset.Batch().Jobs(Namespace).Create(&newjob)
+	if err != nil {
+		log.Error("error creating Job " + err.Error())
+		return
+	}
+	log.Info("created load Job " + resultJob.Name)
+
+}

conf/postgres-operator/csvload-job.json

@@ -66,6 +66,7 @@ BACKUP_STORAGE:
   SUPPLEMENTALGROUPS:  65534
 PGO:
   LSPVC_TEMPLATE:  /home/youruserid/.pgo.lspvc-template.json
+  CSVLOAD_TEMPLATE:  /home/youruserid/.pgo.csvload-template.json
   CO_IMAGE_TAG:  centos7-1.5.1
   DEBUG:  false
 ....
@@ -111,6 +112,7 @@ Values in the pgo configuration file have the following meaning:
 |BACKUP_STORAGE.FSGROUP        | optional, if set, will cause a *SecurityContext* and *fsGroup* attributes to be added to generated Pod and Deployment definitions 
 |BACKUP_STORAGE.SUPPLEMENTAL_GROUPS        | optional, if set, will cause a SecurityContext to be added to generated Pod and Deployment definitions 
 |PGO.LSPVC_TEMPLATE        | the PVC lspvc template file used to view PVCs
+|PGO.CSVLOAD_TEMPLATE        | the CSV load template file used for load jobs
 |PGO.CO_IMAGE_TAG        | image tag to use for the PostgreSQL operator containers
 |PGO.DEBUG        | set to true if you want to see debug messages from the pgo client
 |======================

docs/config.asciidoc

@@ -0,0 +1,17 @@
+  {
+      "kind": "PersistentVolumeClaim",
+      "apiVersion": "v1",
+      "metadata": {
+          "name": "csv-pvc"
+      },
+      "spec": {
+          "accessModes": [
+              "ReadWriteMany"
+          ],
+          "resources": {
+              "requests": {
+                  "storage": "100M"
+              }
+          }
+      }
+  }

examples/operator/csv-pvc.json

@@ -30,5 +30,6 @@ REPLICA_STORAGE:
   SUPPLEMENTAL_GROUPS:  65534
 PGO:
   LSPVC_TEMPLATE:  /home/jeffmc/.pgo.lspvc-template.json
+  CSVLOAD_TEMPLATE:  /home/jeffmc/.pgo.csvload-template.json
   CO_IMAGE_TAG:  centos7-1.5.1
   DEBUG:  false

examples/pgo.yaml.emptydir

@@ -30,5 +30,6 @@ REPLICA_STORAGE:
   SUPPLEMENTAL_GROUPS:  65534
 PGO:
   LSPVC_TEMPLATE:  /home/jeffmc/.pgo.lspvc-template.json
+  CSVLOAD_TEMPLATE:  /home/jeffmc/.pgo.csvload-template.json
   CO_IMAGE_TAG:  centos7-1.5.1
   DEBUG:  false

examples/pgo.yaml.nfs

@@ -33,5 +33,6 @@ REPLICA_STORAGE:
   FSGROUP:  26
 PGO:
   LSPVC_TEMPLATE:  /home/jeffmc/.pgo.lspvc-template.json
+  CSVLOAD_TEMPLATE:  /home/jeffmc/.pgo.csvload-template.json
   CO_IMAGE_TAG:  centos7-1.5.1
   DEBUG:  false

examples/pgo.yaml.storageclass

@@ -1,3 +1,5 @@
 \c userdb;
 create table xrayapp (id int, key string, value string);
 create table a (id int);
+create table xraycsvtable (name string, state string, zip string);
+