Merge pull request #4 from rustprooflabs/polish-initial-functionality

Cleanup and streamline

Author: rustprooflabs

Dockerfile

@@ -1,4 +1,6 @@
 FROM rustprooflabs/pgosm-flex
 
 COPY ./db /app/faker/db
-
+COPY ./faker.ini /app/flex-config/layerset/
+COPY ./run_faker.sh /app/
+COPY ./run_faker.sql /app/

db/deploy/001.sql

@@ -2,10 +2,10 @@
 
 BEGIN;
 
-CREATE SCHEMA geo_faker;
+CREATE SCHEMA geofaker;
 
 
-CREATE PROCEDURE geo_faker.point_in_place_landuse()
+CREATE PROCEDURE geofaker.point_in_place_landuse()
 LANGUAGE plpgsql
 AS $$
 BEGIN
@@ -223,7 +223,7 @@ BEGIN
 
 	DROP TABLE IF EXISTS faker_store_location;
 	CREATE TEMP TABLE faker_store_location AS
-	SELECT ROW_NUMBER() OVER () AS id, a.place_osm_id, a.place_osm_type, a.place_name, a.road_osm_id,
+	SELECT ROW_NUMBER() OVER () AS store_id, a.place_osm_id, a.place_osm_type, a.place_name, a.road_osm_id,
 			r.osm_type AS road_osm_type, r.name AS road_name, r.ref AS road_ref,
 			public.ST_LineInterpolatePoint(public.ST_LineMerge(r.geom), random()) AS geom
 		FROM selected_roads a
@@ -235,13 +235,13 @@ END
 $$
 ;
 
-COMMENT ON PROCEDURE geo_faker.point_in_place_landuse IS 'Uses osm.landuse_polygon and osm.road_line to simulate probable locations for commercial store locations.  Can be customized for custom landuse types by manually defining landuse_osm_types temp table.'
+COMMENT ON PROCEDURE geofaker.point_in_place_landuse IS 'Uses osm.landuse_polygon and osm.road_line to simulate probable locations for commercial store locations.  Can be customized for custom landuse types by manually defining landuse_osm_types temp table.'
 ;
 
 
 -- From: https://trac.osgeo.org/postgis/wiki/UserWikiRandomPoint
 
-CREATE FUNCTION geo_faker.n_points_in_polygon(geom geometry, num_points integer)
+CREATE FUNCTION geofaker.n_points_in_polygon(geom geometry, num_points integer)
   	RETURNS SETOF geometry 
     LANGUAGE plpgsql VOLATILE
   	COST 100
@@ -288,51 +288,57 @@ END
 $$
 ;
 
-COMMENT ON FUNCTION geo_faker.n_points_in_polygon(GEOMETRY, INT) IS 'Creates N points randomly within the given polygon.';
+COMMENT ON FUNCTION geofaker.n_points_in_polygon(GEOMETRY, INT) IS 'Creates N points randomly within the given polygon.';
 
 
 
--- Call the procedure to ensure the required temp table exists, avoids deploy failure
-CALL geo_faker.point_in_place_landuse();
+-- Ensure the required temp table exists, avoids deploy failure on next sproc
+CREATE TEMP TABLE IF NOT EXISTS faker_store_location
+(
+	store_id BIGINT, place_osm_id BIGINT, place_osm_type TEXT, place_name TEXT,
+	road_osm_id BIGINT, geom GEOMETRY
+);
 
 
-CREATE PROCEDURE geo_faker.points_around_point()
- LANGUAGE plpgsql
- AS $$
- DECLARE
-    stores_to_process BIGINT;
+CREATE PROCEDURE geofaker.points_around_point()
+	LANGUAGE plpgsql
+AS $$
+DECLARE
+	stores_to_process BIGINT;
 	t_row faker_store_location%rowtype;
- BEGIN
+BEGIN
 
- SELECT  COUNT(*) INTO stores_to_process
-     FROM faker_store_location
- ;
- RAISE NOTICE 'Stores to process: %', stores_to_process;
+	SELECT  COUNT(*) INTO stores_to_process
+		FROM faker_store_location
+	;
+	RAISE NOTICE 'Generating customers for % stores...', stores_to_process;
 
-DROP TABLE IF EXISTS faker_customer_location;
-CREATE TEMP TABLE faker_customer_location
-(
-	id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
-	store_id BIGINT NOT NULL,
-	customer_id BIGINT NOT NULL,
-	geom GEOMETRY(POINT, 3857) NOT NULL
-);
+	DROP TABLE IF EXISTS faker_customer_location;
+	CREATE TEMP TABLE faker_customer_location
+	(
+		id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+		store_id BIGINT NOT NULL,
+		customer_id BIGINT NOT NULL,
+		geom GEOMETRY(POINT, 3857) NOT NULL
+	);
 
 
 	FOR t_row IN SELECT * FROM faker_store_location LOOP
-		RAISE NOTICE 'Store ID: %', t_row.id;
+		IF t_row.store_id % 10 = 0 THEN
+			RAISE NOTICE 'Store ID: %', t_row.store_id;
+		END IF;
 
 		DROP TABLE IF EXISTS place_buffer;
 		CREATE TEMP TABLE place_buffer AS
-		SELECT id AS store_id, geom, ST_Buffer(geom, 5000) AS geom_buffer
+		SELECT store_id, geom, ST_Buffer(geom, 5000) AS geom_buffer
 			FROM faker_store_location
-		    WHERE id = t_row.id
+			WHERE store_id = t_row.store_id
 		;
 
 		DROP TABLE IF EXISTS store_potential_customers;
 		CREATE TEMP TABLE store_potential_customers AS
 		SELECT store_id,
-				geo_faker.n_points_in_polygon(geom_buffer, 1000)
+				geofaker.n_points_in_polygon(geom_buffer, 1000)
 					AS geom
 			FROM place_buffer
 		;
@@ -374,7 +380,7 @@ END;
 $$;
 
 
-COMMENT ON PROCEDURE geo_faker.points_around_point IS 'Creates fake customer locations around a store.  Locations are snapped to roads.  Locations not scoped to landuse at this time.  Requires faker_store_location temp table with fake store data.';
+COMMENT ON PROCEDURE geofaker.points_around_point IS 'Creates fake customer locations around a store.  Locations are snapped to roads.  Locations not scoped to landuse at this time.  Requires faker_store_location temp table with fake store data.';
 
 COMMIT;
 

db/revert/001.sql

@@ -2,6 +2,6 @@
 
 BEGIN;
 
-DROP SCHEMA geo_faker CASCADE;
+DROP SCHEMA geofaker CASCADE;
 
 COMMIT;

docs/book.toml

@@ -8,5 +8,5 @@ title = "Geo Faker"
 [output.html]
 default-theme = "rust"
 preferred-dark-theme = "navy"
-git-repository-url = "https://github.com/rustprooflabs/geo-faker"
+git-repository-url = "https://github.com/rustprooflabs/geofaker"
 git-repository-icon = "fa-github"

docs/src/customize.md

@@ -5,23 +5,6 @@ This section builds on the [Quick Start](quick-start.md) section.
 > Warning: This project is in early development!  Things will be changing over the first few releases (e.g. before 0.5.0).
 
 
-## External Postgres connections
-
-The quick start uses the built-in Postgres/PostGIS instance. See
-the PgOSM Flex section on [Using external Postgres connections](https://pgosm-flex.com/postgres-external.html) to use your own Postgres instance.
-This approach does load a lot of data to the target database which may not be
-desired.  Consider using `pg_dump` to load only the target data to your
-database of choice.
-
-The Sqitch deployment step should use additional parameters not set in the quick start
-instructions.
-
-```bash
-source ~/.pgosm-faker-local
-cd pgosm-flex-faker/db
-sqitch db:pg://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB deploy
-```
-
 
 ## Each time is new data
 
@@ -54,3 +37,13 @@ UNION
 SELECT 'vineyard' AS osm_type
 ;
 ```
+
+
+## External Postgres connections
+
+Geo Faker currently does not support running directly into an external database.
+Technically it can be operated that way, but the instructions and helper scripts
+are specific to using the in-Docker database.
+
+Use `--pg-dump` to extract the generated data.
+

docs/src/docker-image.md

@@ -8,10 +8,10 @@ Build latest.  Occasionally run with `--no-cache` to force some software updates
 
 ```bash
 docker pull rustprooflabs/pgosm-flex:latest
-docker build -t rustprooflabs/geo-faker:latest .
+docker build -t rustprooflabs/geofaker:latest .
 ```
 
 
 ```bash
-docker push rustprooflabs/geo-faker:latest
+docker push rustprooflabs/geofaker:latest
 ```

docs/src/quick-start.md

@@ -19,7 +19,7 @@ The basic process to using Geo Faker are:
 Load the region/subregion you want using the PgOSM Flex Docker image.
 These instructions are modified from
 [PgOSM Flex's Quick Start](https://pgosm-flex.com/quick-start.html)
-section. The following loads the data into a PostGIS enabled database in a `geo-faker`
+section. The following loads the data into a PostGIS enabled database in a `geofaker`
 Docker container available on port 5433.
 
 
@@ -28,84 +28,39 @@ mkdir ~/pgosm-data
 export POSTGRES_USER=postgres
 export POSTGRES_PASSWORD=mysecretpassword
 
-docker pull rustprooflabs/geo-faker:latest
+docker pull rustprooflabs/geofaker:latest
 
-docker run --name pgosm-faker -d --rm \
+docker run --name geofaker -d --rm \
     -v ~/pgosm-data:/app/output \
-    -v ~/git/geo-faker/:/custom-layerset \
     -v /etc/localtime:/etc/localtime:ro \
+    -e POSTGRES_USER=$POSTGRES_USER \
     -e POSTGRES_PASSWORD=$POSTGRES_PASSWORD \
-    -p 5433:5432 -d rustprooflabs/geo-faker
+    -p 5433:5432 -d rustprooflabs/geofaker
 
 docker exec -it \
-    pgosm-faker python3 docker/pgosm_flex.py \
+    geofaker python3 docker/pgosm_flex.py \
     --ram=8 \
     --region=north-america/us \
     --subregion=nebraska \
-    --layerset=faker_layerset \
-    --layerset-path=/custom-layerset/
+    --layerset=faker
 ```
 
 
-## Load Faker Objects
+## Load and Run Faker Objects
 
 After the data completes processing, load the Geo Faker database structures
-in the `geo_faker` schema.
-This is done using Sqitch.
+in the `geofaker` schema.  This deploys the functions and procedures needed,
+runs the processing, and runs `pg_dump` saving the faked
+data into `~/pgosm-data/geofaker_stores_customers.sql`.
 
 
 ```bash
-cd geo-faker/db
-sqitch db:pg://$POSTGRES_USER:$POSTGRES_PASSWORD@localhost:5433/pgosm deploy
+docker exec -it geofaker /bin/bash run_faker.sh
 ```
 
-Connect to the database and call this stored procedure.  The generated data
-is left in a temp table, each run of the stored procedure will produce new,
-random results.
+Load the saved data into a database of your choice.
 
-## Run Faker generation
-
-There are two stored procedures in the `geo_faker` schema that
-generate the fake stores and customers.
-
-
-The stored procedure `geo_faker.point_in_place_landuse()` places points
-along roads that are within (or nearby) specific `landuse` areas.  The generated
-data is available after calling the stored procedure in a temporary table
-named `faker_store_location`.
-The generated data is scoped to named places currently, though that will
-likely become adjustable in the future.
-
-
-The `geo_faker.point_in_place_landuse()` stored procedure requires
-the `faker_store_location` temp table created by the first stored procedure.
-
-
-
-```sql
-CALL geo_faker.point_in_place_landuse();
-CALL geo_faker.points_around_point();
-```
-
-The following query saves the data in a new, non-temporary table named
-`my_fake_stores`.
-
-
-
-
-
-
-```sql
-CREATE TABLE my_fake_stores AS
-SELECT *
-    FROM faker_store_location
-;
-
-CREATE TABLE my_fake_customers AS
-SELECT *
-    FROM faker_customer_location
-;
+```bash
+psql -d pgosm_faker -f ~/pgosm-data/geofaker_stores_customers.sql 
 ```
 
-
-

faker_layerset.ini renamed to faker.ini

@@ -0,0 +1,12 @@
+#!/bin/bash
+echo 'Deploying geofaker schema via Sqitch...'
+cd /app/faker/db
+sqitch db:pg://$POSTGRES_USER:$POSTGRES_PASSWORD@localhost/pgosm deploy
+
+echo 'Running GeoFaker generation...'
+psql -d postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@localhost/pgosm \
+    -f /app/run_faker.sql
+
+echo 'Running pg_dump...'
+pg_dump -d postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@localhost/pgosm \
+    -n geofaker -f /app/output/geofaker_stores_customers.sql

run_faker.sh

@@ -0,0 +1,22 @@
+CALL geofaker.point_in_place_landuse();
+CALL geofaker.points_around_point();
+
+
+DROP TABLE IF EXISTS geofaker.store;
+CREATE TABLE geofaker.store AS
+SELECT *
+    FROM faker_store_location
+    ORDER BY store_id
+;
+COMMENT ON TABLE geofaker.store IS 'Created by Geo Faker, a PgOSM Flex based project.';
+
+DROP TABLE IF EXISTS geofaker.customer;
+CREATE TABLE geofaker.customer AS
+SELECT *
+    FROM faker_customer_location
+    ORDER BY store_id, customer_id
+;
+COMMENT ON TABLE geofaker.customer IS 'Created by Geo Faker, a PgOSM Flex based project.';
+
+
+