wip

Author: martijnvans

Makefile

@@ -64,7 +64,7 @@ docker_test_integration:
 		-e SERVICE_ACCOUNT_JSON \
 		-v $(CURDIR):/workspace \
 		$(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
-		/usr/local/bin/test_integration.sh
+		cft test run all
 
 # Execute lint tests within the docker container
 .PHONY: docker_test_lint

build/int.cloudbuild.yaml

@@ -24,19 +24,11 @@ steps:
   - 'TF_VAR_org_id=$_ORG_ID'
   - 'TF_VAR_folder_id=$_FOLDER_ID'
   - 'TF_VAR_billing_account=$_BILLING_ACCOUNT'
-- id: create
+- id: multiple-tables
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create']
-- id: converge
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge']
-- id: verify
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify']
-- id: destroy
-  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
-  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy']
+  args: ['/bin/bash', '-c', 'cft test run TestMultipleTables --verbose']
 - id: create-dwh
+  waitFor: prepare
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestDataWarehouse --stage init --verbose']
 - id: apply-dwh

test/integration/data_warehouse/data_warehouse_test.go

@@ -66,28 +66,28 @@ func TestDataWarehouse(t *testing.T) {
 		utils.Poll(t, verifyWorkflows, 8, 30*time.Second)
 
 		homeDir, err := os.UserHomeDir()
-        if err != nil {
-            log.Fatal(err)
-        }
-        file, err := os.Create(homeDir + "/.bigqueryrc")
-        if err != nil {
-            log.Fatal(err)
-        }
-        file.Close()
+		if err != nil {
+			log.Fatal(err)
+		}
+		file, err := os.Create(homeDir + "/.bigqueryrc")
+		if err != nil {
+			log.Fatal(err)
+		}
+		file.Close()
 
 		// Assert BigQuery tables & views are not empty
-		test_tables := func (){
+		test_tables := func() {
 
 			tables := []string{
-			"thelook.distribution_centers",
-			"thelook.events",
-			"thelook.inventory_items",
-			"thelook.order_items",
-			"thelook.orders",
-			"thelook.products",
-			"thelook.users",
-			"thelook.lookerstudio_report_distribution_centers",
-			"thelook.lookerstudio_report_profit",
+				"thelook.distribution_centers",
+				"thelook.events",
+				"thelook.inventory_items",
+				"thelook.order_items",
+				"thelook.orders",
+				"thelook.products",
+				"thelook.users",
+				"thelook.lookerstudio_report_distribution_centers",
+				"thelook.lookerstudio_report_profit",
 			}
 
 			query_template := "SELECT COUNT(*) AS count_rows FROM `%[1]s.%[2]s`;"
@@ -108,7 +108,6 @@ func TestDataWarehouse(t *testing.T) {
 
 		// Assert BigQuery connection to Vertex GenAI was successfully created and works as expected
 		test_llms := func() {
-
 			llm_query_template := "SELECT COUNT(*) AS count_rows FROM ML.GENERATE_TEXT(MODEL `%[1]s.thelook.text_generate_model`, (with clusters AS(SELECT CONCAT('cluster', CAST(centroid_id as STRING)) as centroid, avg_spend as average_spend, count_orders as count_of_orders, days_since_order FROM (SELECT centroid_id, feature, ROUND(numerical_value, 2) as value FROM ML.CENTROIDS(MODEL `%[1]s.thelook.customer_segment_clustering`)) PIVOT (SUM(value) FOR feature IN ('avg_spend', 'count_orders', 'days_since_order')) ORDER BY centroid_id) SELECT 'Pretend you are a creative strategist, given the following clusters come up with creative brand persona and title labels for each of these clusters, and explain step by step; what would be the next marketing step for these clusters' || ' ' || clusters.centroid || ', Average Spend $' || clusters.average_spend || ', Count of orders per person ' || clusters.count_of_orders || ', Days since last order ' || clusters.days_since_order AS prompt FROM clusters), STRUCT(800 AS max_output_tokens, 0.8 AS temperature, 40 AS top_k, 0.8 AS top_p, TRUE AS flatten_json_output));"
 			query := fmt.Sprintf(llm_query_template, projectID)
 			llm_op := bq.Runf(t, "--project_id=%[1]s --headless=true query --nouse_legacy_sql %[2]s", projectID, query)
@@ -117,9 +116,9 @@ func TestDataWarehouse(t *testing.T) {
 			count_llm_kind := reflect.TypeOf(llm_count).Kind()
 			llm_test_result := assert.Greater(llm_count, int64(0))
 			if llm_test_result == true {
-				} else {
-					fmt.Printf("Some kind of error occurred while running the llm_count query. We think it has %[1]d rows. Here's some additional details: \n Query results. If this number is greater than 0, then there is probably an issue in the comparison: %[2]s \n Variable type for the count. This should be INT64: %[3]s \n ", llm_count, llm_op, count_llm_kind)
-				}
+			} else {
+				fmt.Printf("Some kind of error occurred while running the llm_count query. We think it has %[1]d rows. Here's some additional details: \n Query results. If this number is greater than 0, then there is probably an issue in the comparison: %[2]s \n Variable type for the count. This should be INT64: %[3]s \n ", llm_count, llm_op, count_llm_kind)
+			}
 		}
 		test_llms()
 	})