fixed shell scripts

Author: joshplotkin

datagen_HAWQ/README.md

@@ -4,3 +4,8 @@
 * install fake-factory (tarball included) and numpy. you can use `pip install <package>` or unzip the package tarballs and run `python setup.py build; python setup.py install` in the package directory.
 * to test usage, run 'python EXECUTE.py' for to generate 1 customer's transactions
 * to create a dataset trans_fact.csv, run 'sh GENERATE_DATASET.sh N' where N is the desired number of customers
+* to run in HAWQ
+  * move this directory to the same location on each HAWQ host 
+  * run ./process_datagen.sh <# of customers> to generate data in HAWQ ('trans_fact' table and 'spending' view)
+  * roughly 2000 customers per 1 GB of data as a default
+* note/TODO: as a default, the travel values are set too high

datagen_HAWQ/hawq.sql

@@ -46,20 +46,20 @@ amt float8)
 DISTRIBUTED RANDOMLY;
 
 DROP FUNCTION IF EXISTS customer(integer, text) CASCADE;
-CREATE OR REPLACE FUNCTION customer(repeat text)
+CREATE OR REPLACE FUNCTION customer(repeat text, dir text)
 	RETURNS SETOF transaction
 AS
-$$  
+$$
 	import cPickle as pickle
 	import faker
 	from faker import Faker
 	import numpy as np
 	import sys
 	import os
 
-	sys.path.append('/demo/103_datagen_HAWQ/datagen_HAWQ/src')
-	sys.path.append('/demo/103_datagen_HAWQ/datagen_HAWQ')
-	os.chdir('/demo/103_datagen_HAWQ/datagen_HAWQ')
+	sys.path.append(dir + '/src')
+	sys.path.append(dir)
+	os.chdir(dir)
 
 	# not currently working: repeatable data set
 	# if repeat.lower() != 'False':
@@ -72,7 +72,7 @@ $$
 	#for a in alltrans:
 	#	yield(a)
 	return [a for a in alltrans]
-$$ 
+$$
 LANGUAGE PLPYTHONU;
 
 -- [pivhdsne:datagen_HAWQ]$ time psql -f hawq.sql -v customers=2
@@ -82,55 +82,55 @@ SELECT generate_series(1, :customers);
 
 -- TODO: look into seed
 INSERT INTO trans_fact
-SELECT (t).* 
-FROM 
-	(SELECT (customer('False')) as t
+SELECT (t).*
+FROM
+	(SELECT (customer('False', :pwd)) as t
 	FROM (
-		SELECT n 
-		FROM numbers 
+		SELECT n
+		FROM numbers
 		)q
 	)q2;
 
 CREATE VIEW spending AS (
-SELECT ssn, age, gender, count(*) as num_trans, 
-	sum(food_dining) as food_dining, 
-	sum(utilities) as utilities, 
-	sum(grocery_net) as grocery_net, 
-	sum(home) as home, 
-	sum(pharmacy) as pharmacy, 
-	sum(shopping_pos) as shopping_pos, 
-	sum(kids_pets) as kids_pets, 
-	sum(personal_care) as personal_care, 
-	sum(misc_pos) as misc_pos, 
-	sum(gas_transport) as gas_transport, 
-	sum(misc_net) as misc_net, 
-	sum(health_fitness) as health_fitness, 
-	sum(shopping_net) as shopping_net, 
+SELECT ssn, age, gender, count(*) as num_trans,
+	sum(food_dining) as food_dining,
+	sum(utilities) as utilities,
+	sum(grocery_net) as grocery_net,
+	sum(home) as home,
+	sum(pharmacy) as pharmacy,
+	sum(shopping_pos) as shopping_pos,
+	sum(kids_pets) as kids_pets,
+	sum(personal_care) as personal_care,
+	sum(misc_pos) as misc_pos,
+	sum(gas_transport) as gas_transport,
+	sum(misc_net) as misc_net,
+	sum(health_fitness) as health_fitness,
+	sum(shopping_net) as shopping_net,
 	sum(travel) as travel
 	FROM(
 	-- create case statements
-	SELECT ssn as ssn, extract(years from age(NOW(),dob)) as age, 
+	SELECT ssn as ssn, extract(years from age(NOW(),dob)) as age,
 	case when gender = 'M' then 0 else 1 end as gender,
 	case when category = 'food_dining' then (amt) else 0 end as food_dining,
-	case when category = 'utilities' then (amt) else 0 end as utilities, 
-	case when category = 'grocery_net' then (amt) else 0 end as grocery_net, 
-	case when category = 'home' then (amt) else 0 end as home, 
-	case when category = 'pharmacy' then (amt) else 0 end as pharmacy, 
-	case when category = 'shopping_pos' then (amt) else 0 end as shopping_pos, 
-	case when category = 'kids_pets' then (amt) else 0 end as kids_pets, 
-	case when category = 'personal_care' then (amt) else 0 end as personal_care, 
-	case when category = 'misc_pos' then (amt) else 0 end as misc_pos, 
-	case when category = 'gas_transport' then (amt) else 0 end as gas_transport, 
-	case when category = 'misc_net' then (amt) else 0 end as misc_net, 
-	case when category = 'health_fitness' then (amt) else 0 end as health_fitness, 
-	case when category = 'shopping_net' then (amt) else 0 end as shopping_net, 
+	case when category = 'utilities' then (amt) else 0 end as utilities,
+	case when category = 'grocery_net' then (amt) else 0 end as grocery_net,
+	case when category = 'home' then (amt) else 0 end as home,
+	case when category = 'pharmacy' then (amt) else 0 end as pharmacy,
+	case when category = 'shopping_pos' then (amt) else 0 end as shopping_pos,
+	case when category = 'kids_pets' then (amt) else 0 end as kids_pets,
+	case when category = 'personal_care' then (amt) else 0 end as personal_care,
+	case when category = 'misc_pos' then (amt) else 0 end as misc_pos,
+	case when category = 'gas_transport' then (amt) else 0 end as gas_transport,
+	case when category = 'misc_net' then (amt) else 0 end as misc_net,
+	case when category = 'health_fitness' then (amt) else 0 end as health_fitness,
+	case when category = 'shopping_net' then (amt) else 0 end as shopping_net,
 	case when category = 'travel' then (amt) else 0 end as travel
 
 	FROM trans_fact) GROUPED
 	GROUP BY ssn, age, gender
 );
 
-VACUUM ANALYZE trans_fact;
+ANALYZE trans_fact;
 
 SELECT nspname || '.' || relname AS "relation",
 	pg_size_pretty(pg_relation_size(C.oid)) AS "size"
@@ -140,5 +140,4 @@ SELECT nspname || '.' || relname AS "relation",
   ORDER BY pg_relation_size(C.oid) DESC
   LIMIT 20;
 
-select now();
-
+select now();

datagen_HAWQ/process_datagen.sh

@@ -4,8 +4,8 @@ if [ -z $1 ]
   then
     echo 'enter number of customers as command line argument'
   else
-    psql -U gpadmin -p 10432 -f hawq.sql -v customers=$1
+    psql -U gpadmin -p 5432 -f hawq.sql -v customers=$1
     export PGPASSWORD=gpadmin
-    psql -p 10432 -U gpadmin -c "VACUUM ANALYZE trans_fact"
+    psql -p 5432 -U gpadmin -c "VACUUM ANALYZE trans_fact"
 fi