Merge pull request #12 from rustprooflabs/add-pgfaker

Improve faked data - pgfaker

Author: rustprooflabs

Dockerfile

@@ -1,6 +1,13 @@
 FROM rustprooflabs/pgosm-flex
 
+# Install pgfaker extension
+RUN wget https://github.com/rustprooflabs/pgfaker/releases/download/0.0.1/pgfaker_0.0.1_debian-11_pg15_amd64.deb \
+        -O /tmp/pgfaker.deb \
+    && dpkg -i --force-overwrite /tmp/pgfaker.deb
+
+
 COPY ./db /app/faker/db
 COPY ./faker.ini /app/flex-config/layerset/
-COPY ./run_faker.sh /app/
-COPY ./run_faker.sql /app/
+
+COPY ./app/* /app/
+

run_faker.sh app/run_faker.shrun_faker.sh renamed to app/run_faker.sh

@@ -2,8 +2,8 @@ CALL geofaker.point_in_place_landuse();
 /*
     Sets scale of distance for customers, and density of customer points 
 */
-CALL geofaker.points_around_point(_distance_scale:=0.5,
-                                  _density_scale:=4);
+CALL geofaker.points_around_point(_distance_scale:=1.5,
+                                  _density_scale:=1);
 
 
 DROP TABLE IF EXISTS geofaker.store;

run_faker.sql app/run_faker.sqlrun_faker.sql renamed to app/run_faker.sql

@@ -0,0 +1,8 @@
+-- Deploy pgosm-flex-faker:extensions to pg
+
+BEGIN;
+
+CREATE SCHEMA IF NOT EXISTS geofaker;
+CREATE EXTENSION IF NOT EXISTS pgfaker;
+
+COMMIT;

db/deploy/basics.sql

@@ -0,0 +1,58 @@
+-- Deploy pgosm-flex-faker:n-points-in-polygon to pg
+-- requires: basics
+
+BEGIN;
+
+
+CREATE FUNCTION geofaker.n_points_in_polygon(geom geometry, num_points integer)
+  	RETURNS SETOF geometry 
+    LANGUAGE plpgsql VOLATILE
+  	COST 100
+  	ROWS 1000
+AS $$
+DECLARE
+	target_proportion numeric;
+	n_ret integer := 0;
+	loops integer := 0;
+	x_min float8;
+	y_min float8;
+	x_max float8;
+	y_max float8;
+	srid integer;
+	rpoint geometry;
+BEGIN
+	-- Get envelope and SRID of source polygon
+	SELECT ST_XMin(geom), ST_YMin(geom), ST_XMax(geom), ST_YMax(geom), ST_SRID(geom)
+		INTO x_min, y_min, x_max, y_max, srid;
+	-- Get the area proportion of envelope size to determine if a
+	-- result can be returned in a reasonable amount of time
+	SELECT ST_Area(geom)/ST_Area(ST_Envelope(geom)) INTO target_proportion;
+	RAISE DEBUG 'geom: SRID %, NumGeometries %, NPoints %, area proportion within envelope %',
+					srid, ST_NumGeometries(geom), ST_NPoints(geom),
+					round(100.0*target_proportion, 2) || '%';
+	IF target_proportion < 0.0001 THEN
+		RAISE EXCEPTION 'Target area proportion of geometry is too low (%)', 
+						100.0*target_proportion || '%';
+	END IF;
+	RAISE DEBUG 'bounds: % % % %', x_min, y_min, x_max, y_max;
+	
+	WHILE n_ret < num_points LOOP
+		loops := loops + 1;
+		SELECT ST_SetSRID(ST_MakePoint(random()*(x_max - x_min) + x_min,
+									random()*(y_max - y_min) + y_min),
+						srid) INTO rpoint;
+		IF ST_Contains(geom, rpoint) THEN
+		n_ret := n_ret + 1;
+		RETURN NEXT rpoint;
+		END IF;
+	END LOOP;
+	RAISE DEBUG 'determined in % loops (% efficiency)', loops, round(100.0*num_points/loops, 2) || '%';
+END
+$$
+;
+
+COMMENT ON FUNCTION geofaker.n_points_in_polygon(GEOMETRY, INT) IS 'Creates N points randomly within the given polygon.  From: https://trac.osgeo.org/postgis/wiki/UserWikiRandomPoint';
+
+
+
+COMMIT;

db/deploy/n-points-in-polygon.sql

@@ -1,9 +1,8 @@
--- Deploy pgosm-flex-faker:001 to pg
+-- Deploy pgosm-flex-faker:point-in-place-landuse to pg
+-- requires: basics
 
 BEGIN;
 
-CREATE SCHEMA geofaker;
-
 
 CREATE PROCEDURE geofaker.point_in_place_landuse()
 LANGUAGE plpgsql
@@ -223,8 +222,9 @@ BEGIN
 
 	DROP TABLE IF EXISTS faker_store_location;
 	CREATE TEMP TABLE faker_store_location AS
-	SELECT ROW_NUMBER() OVER () AS store_id, a.place_osm_id, a.place_osm_type, a.place_name, a.road_osm_id,
-			r.osm_type AS road_osm_type, r.name AS road_name, r.ref AS road_ref,
+	SELECT ROW_NUMBER() OVER () AS store_id, a.place_name AS city,
+			r.name AS street_name, r.ref AS road_ref,
+            pgfaker.company(), pgfaker.slogan(), pgfaker.phone(),
 			public.ST_LineInterpolatePoint(public.ST_LineMerge(r.geom), random()) AS geom
 		FROM selected_roads a
 		INNER JOIN osm.road_line r ON a.road_osm_id = r.osm_id
@@ -239,149 +239,4 @@ COMMENT ON PROCEDURE geofaker.point_in_place_landuse IS 'Uses osm.landuse_polygo
 ;
 
 
-CREATE FUNCTION geofaker.n_points_in_polygon(geom geometry, num_points integer)
-  	RETURNS SETOF geometry 
-    LANGUAGE plpgsql VOLATILE
-  	COST 100
-  	ROWS 1000
-AS $$
-DECLARE
-	target_proportion numeric;
-	n_ret integer := 0;
-	loops integer := 0;
-	x_min float8;
-	y_min float8;
-	x_max float8;
-	y_max float8;
-	srid integer;
-	rpoint geometry;
-BEGIN
-	-- Get envelope and SRID of source polygon
-	SELECT ST_XMin(geom), ST_YMin(geom), ST_XMax(geom), ST_YMax(geom), ST_SRID(geom)
-		INTO x_min, y_min, x_max, y_max, srid;
-	-- Get the area proportion of envelope size to determine if a
-	-- result can be returned in a reasonable amount of time
-	SELECT ST_Area(geom)/ST_Area(ST_Envelope(geom)) INTO target_proportion;
-	RAISE DEBUG 'geom: SRID %, NumGeometries %, NPoints %, area proportion within envelope %',
-					srid, ST_NumGeometries(geom), ST_NPoints(geom),
-					round(100.0*target_proportion, 2) || '%';
-	IF target_proportion < 0.0001 THEN
-		RAISE EXCEPTION 'Target area proportion of geometry is too low (%)', 
-						100.0*target_proportion || '%';
-	END IF;
-	RAISE DEBUG 'bounds: % % % %', x_min, y_min, x_max, y_max;
-	
-	WHILE n_ret < num_points LOOP
-		loops := loops + 1;
-		SELECT ST_SetSRID(ST_MakePoint(random()*(x_max - x_min) + x_min,
-									random()*(y_max - y_min) + y_min),
-						srid) INTO rpoint;
-		IF ST_Contains(geom, rpoint) THEN
-		n_ret := n_ret + 1;
-		RETURN NEXT rpoint;
-		END IF;
-	END LOOP;
-	RAISE DEBUG 'determined in % loops (% efficiency)', loops, round(100.0*num_points/loops, 2) || '%';
-END
-$$
-;
-
-COMMENT ON FUNCTION geofaker.n_points_in_polygon(GEOMETRY, INT) IS 'Creates N points randomly within the given polygon.  From: https://trac.osgeo.org/postgis/wiki/UserWikiRandomPoint';
-
-
-
--- Ensure the required temp table exists, avoids deploy failure creating next sproc
-CREATE TEMP TABLE IF NOT EXISTS faker_store_location
-(
-	store_id BIGINT, place_osm_id BIGINT, place_osm_type TEXT, place_name TEXT,
-	road_osm_id BIGINT, geom GEOMETRY
-);
-
-
-CREATE PROCEDURE geofaker.points_around_point()
-	LANGUAGE plpgsql
-AS $$
-DECLARE
-	stores_to_process BIGINT;
-	t_row faker_store_location%rowtype;
-BEGIN
-
-	SELECT  COUNT(*) INTO stores_to_process
-		FROM faker_store_location
-	;
-	RAISE NOTICE 'Generating customers for % stores...', stores_to_process;
-
-	DROP TABLE IF EXISTS faker_customer_location;
-	CREATE TEMP TABLE faker_customer_location
-	(
-		id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
-		store_id BIGINT NOT NULL,
-		customer_id BIGINT NOT NULL,
-		geom GEOMETRY(POINT, 3857) NOT NULL
-	);
-
-
-	FOR t_row IN SELECT * FROM faker_store_location LOOP
-		IF t_row.store_id % 10 = 0 THEN
-			RAISE NOTICE 'Store ID: %', t_row.store_id;
-		END IF;
-
-		DROP TABLE IF EXISTS place_buffer;
-		CREATE TEMP TABLE place_buffer AS
-		SELECT store_id, geom, ST_Buffer(geom, 5000) AS geom_buffer
-			FROM faker_store_location
-			WHERE store_id = t_row.store_id
-		;
-
-		DROP TABLE IF EXISTS store_potential_customers;
-		CREATE TEMP TABLE store_potential_customers AS
-		SELECT store_id,
-				geofaker.n_points_in_polygon(geom_buffer, 1000)
-					AS geom
-			FROM place_buffer
-		;
-		ALTER TABLE store_potential_customers
-			ADD customer_id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY;
-
-		--SELECT * FROM store_potential_customers;
-		/*
-		* Using a CTE here with ST_Envelope to bbox join roads.
-		* A simple join (which looks innocent) took 45+ seconds to return 141 rows
-		* while the CTE version takes < 60 ms.
-		*/
-		--EXPLAIN (ANALYZE, BUFFERS, VERBOSE, SETTINGS)
-		WITH possible_roads AS (
-		SELECT p.store_id, p.customer_id, p.geom AS geom_customer,
-				r.geom AS geom_road,
-				ST_Distance(p.geom, r.geom) AS distance
-			FROM osm.road_line r
-			INNER JOIN store_potential_customers p
-				ON ST_DWithin(r.geom, p.geom, 300)
-			WHERE r.route_motor
-		), ranked AS (
-		SELECT *, ROW_NUMBER() OVER (
-					PARTITION BY store_id, customer_id ORDER BY distance
-					) AS rnk
-			FROM possible_roads
-		)
-		INSERT INTO faker_customer_location (store_id, customer_id, geom)
-		SELECT store_id, customer_id,
-				ST_Snap(geom_customer, geom_road, 300) AS geom_snapped
-			FROM ranked
-			WHERE rnk = 1
-		;
-		COMMIT;
-
-	END LOOP;
-
-END;
-$$;
-
-
-COMMENT ON PROCEDURE geofaker.points_around_point IS 'Creates fake customer locations around a store.  Locations are snapped to roads.  Locations not scoped to landuse at this time.  Requires faker_store_location temp table with fake store data.';
-
 COMMIT;
-
-
-
-

db/deploy/001.sql db/deploy/point-in-place-landuse.sqldb/deploy/001.sql renamed to db/deploy/point-in-place-landuse.sql

@@ -1,5 +1,5 @@
--- Deploy pgosm-flex-faker:002 to pg
--- requires: 001
+-- Deploy pgosm-flex-faker:points-around-point to pg
+-- requires: n-points-in-polygon
 
 BEGIN;
 
@@ -12,9 +12,6 @@ CREATE TEMP TABLE IF NOT EXISTS faker_store_location
 );
 
 
-
-
-DROP PROCEDURE geofaker.points_around_point();
 CREATE PROCEDURE geofaker.points_around_point(
         _distance_scale NUMERIC = 1.0,
         _density_scale NUMERIC = 1.0
@@ -37,6 +34,9 @@ BEGIN
 		id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
 		store_id BIGINT NOT NULL,
 		customer_id BIGINT NOT NULL,
+		full_name TEXT NOT NULL,
+		email TEXT NOT NULL,
+		phone TEXT NULL,
 		geom GEOMETRY(POINT, 3857) NOT NULL
 	);
 
@@ -100,8 +100,11 @@ BEGIN
 					) AS rnk
 			FROM possible_roads
 		)
-		INSERT INTO faker_customer_location (store_id, customer_id, geom)
+		INSERT INTO faker_customer_location (store_id, customer_id,
+				full_name, email, phone, geom)
 		SELECT store_id, customer_id,
+				pgfaker.person_full_name(), pgfaker.email(),
+        		pgfaker.phone(),
 				ST_Snap(geom_customer, geom_road, 300) AS geom_snapped
 			FROM ranked
 			WHERE rnk = 1
@@ -116,8 +119,4 @@ END;
 $$;
 
 
-
-
-
-
 COMMIT;