Merge pull request #183 from mapswipe/dev

fix NoneType because of no users in database

Author: Hagellach37

mapswipe_workers/mapswipe_workers/firebase_to_postgres/update_data.py

@@ -20,15 +20,16 @@ def update_user_data(user_ids=None):
         ORDER BY created DESC
         LIMIT 1
         '''
-    last_updated = pg_db.retr_query(pg_query)
-    if not last_updated:
+    last_updated = pg_db.retr_query(pg_query)[0][0]
+    logger.info(f'got last updated timestamp: {last_updated}')
+
+    if last_updated is None:
         # No users in the Postgres database yet.
         # Get all users from Firebase.
         users = fb_ref.get()
         print(users)
     else:
         # Get only new users from Firebase.
-        last_updated = last_updated[0][0]
         last_updated = last_updated.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
         fb_query = fb_ref.order_by_child('created').start_at(last_updated)
         users = fb_query.get()

postgres/scripts/v1_to_v2/copy_groups_from_csv.sql

@@ -5,7 +5,8 @@
 
 CREATE TEMP TABLE v1_groups (
     project_id varchar,
-    group_id int,
+    v1_group_id int,
+    group_id varchar DEFAULT NULL,
     number_of_tasks int,
     finished_count int,
     required_count int,
@@ -14,7 +15,7 @@ CREATE TEMP TABLE v1_groups (
 );
 
 -- Has to be in one line otherwise syntax error
-\copy v1_groups(project_id, group_id, number_of_tasks, finished_count, required_count, project_type_specifics) FROM groups.csv WITH (FORMAT CSV, DELIMITER ',', HEADER TRUE);
+\copy v1_groups(project_id, v1_group_id, number_of_tasks, finished_count, required_count, project_type_specifics) FROM groups.csv WITH (FORMAT CSV, DELIMITER ',', HEADER TRUE);
 
 -- (Convert old to new data structure).
 -- verification_count (old name for required_count) was static.

postgres/scripts/v1_to_v2/copy_tasks_from_csv.sql

@@ -5,14 +5,18 @@
 
 CREATE TEMP TABLE v1_tasks (
     project_id varchar,
-    group_id int,
+    v1_group_id int,
+    group_id varchar DEFAULT NULL,
     task_id varchar,
     project_type_specifics json,
     geom geometry(MULTIPOLYGON,4326) DEFAULT NULL
 );
 
 -- Has to be in one line otherwise syntax error
-\copy v1_tasks(project_id, group_id, task_id, project_type_specifics) FROM tasks.csv WITH (FORMAT CSV, DELIMITER ',', HEADER TRUE);
+\copy v1_tasks(project_id, v1_group_id, task_id, project_type_specifics) FROM tasks.csv WITH (FORMAT CSV, DELIMITER ',', HEADER TRUE);
+
+UPDATE v1_tasks
+SET group_id = cast(v1_group_id as varchar);
 
 /* Compute geometry of tasks from x, y tile coordinates and z zoom level stored in task id*/
 /* More details on the coordinates: */

postgres/scripts/v1_to_v2/copy_to_csv.sql

@@ -2,5 +2,5 @@
 -- Rename attributes to conform to v2.
 \copy (SELECT archive, image, importkey as "import_id", isfeatured AS "is_featured", lookfor AS "look_for", name, progress, projectdetails AS "project_details", project_id, project_type, state AS "status", info AS "project_type_specifics" FROM projects WHERE project_id in (select project_id from results group by project_id) AND project_id = 5549) TO projects.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
 \copy (SELECT i.import_id, i.info FROM imports i , projects p WHERE import_id in (select importkey as import_id from projects) AND p.project_id = 5549 AND p.importkey = i.import_id) TO imports.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
-\copy (SELECT project_id, group_id, count as "number_of_tasks", completedcount as "finished_count", verificationcount as "required_count", info as "project_type_specifics" FROM groups WHERE project_id in (select project_id from results group by project_id) AND project_id = 5549 ) TO groups.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
-\copy (SELECT project_id, group_id, task_id, info as "project_type_specifics" FROM tasks WHERE project_id in (select project_id from results group by project_id) AND project_id = 5549) TO tasks.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
+\copy (SELECT project_id, group_id as "v1_group_id", count as "number_of_tasks", completedcount as "finished_count", verificationcount as "required_count", info as "project_type_specifics" FROM groups WHERE project_id in (select project_id from results group by project_id) AND project_id = 5549 ) TO groups.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
+\copy (SELECT project_id, group_id as "v1_group_id", task_id, info as "project_type_specifics" FROM tasks WHERE project_id in (select project_id from results group by project_id) AND project_id = 5549) TO tasks.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);