Merge branch 'dev' of github.com:mapswipe/python-mapswipe-workers into dev

Author: Hagellach37

postgres/scripts/v1_to_v2/.gitignore

@@ -0,0 +1 @@
+*.csv

postgres/scripts/v1_to_v2/copy_from_csv.sh

@@ -5,7 +5,14 @@ NAME="mapswipe"
 
 for entity in projects groups tasks users results
 do
-    psql -h localhost -p 5432 -U ${USER} -d ${NAME} -a -f copy_${entity}_from_csv.sql
+    psql \
+        --host localhost \
+        --port 5432 \
+        --username ${USER} \
+        --dbname ${NAME} \
+        --echo-errors \
+        --log-file copy_from_csv.log \
+        --file copy_${entity}_from_csv.sql
 done
 
 # Commands if docker postgres port is not exposed:

postgres/scripts/v1_to_v2/copy_groups_from_csv.sql

@@ -6,7 +6,7 @@
 CREATE TEMP TABLE v1_groups (
     project_id varchar,
     v1_group_id int,
-    group_id varchar DEFAULT NULL,
+    group_id varchar,
     number_of_tasks int,
     finished_count int,
     required_count int,
@@ -29,12 +29,12 @@ WHERE required_count <= 0;
 UPDATE v1_groups
 SET group_id = cast(v1_group_id as varchar);
 
-UPDATE groups
-SET (finished_count, required_count, progress) =
-        (SELECT finished_count, required_count, progress
-        FROM v1_groups
-        WHERE groups.group_id = v1_groups.group_id
-        AND groups.project_id = v1_groups.project_id);
+/* UPDATE groups */
+/* SET (finished_count, required_count, progress) = */
+/*         (SELECT finished_count, required_count, progress */
+/*         FROM v1_groups */
+/*         WHERE groups.group_id = v1_groups.group_id */
+/*         AND groups.project_id = v1_groups.project_id); */
 
 -- Insert or update data of temp table to the permant table.
 -- Note that the special excluded table is used to

postgres/scripts/v1_to_v2/copy_projects_from_csv.sql

@@ -55,9 +55,27 @@ FROM v1_imports
 WHERE v1_projects.import_id = v1_imports.import_id;
 
 -- Convert geometry to postgis geometry type.
-UPDATE v1_projects
-SET geom = ST_Force2D(ST_Multi(ST_GeomFromKML(kml)))
-WHERE kml IS NOT NULL;
+-- If Geoemtry is invalid (Exception gets raised)
+-- Do Nothing. Continue with the next project.
+do $$
+DECLARE
+    r record;
+BEGIN
+    -- record is a structure that contains an element for each column in the select list
+    FOR r IN SELECT * from v1_projects
+    LOOP
+        BEGIN
+            UPDATE v1_projects
+            SET geom = ST_Force2D(ST_Multi(ST_GeomFromKML(kml)))
+            WHERE v1_projects.project_id = r.project_id;
+            -- note the where condition that uses the value from the record variable
+        EXCEPTION
+            WHEN OTHERS THEN
+            -- do nothing
+            END;
+    END LOOP;
+END
+$$ LANGUAGE plpgsql;
 
 -- Insert or update data of temp table to the permant table (projects)
 -- Note that the special excluded table is used to reference values originally proposed for insertion

postgres/scripts/v1_to_v2/copy_results_to_csv.sql

@@ -1,4 +1,4 @@
 -- Export v1 MapSwipe data to csv.
 -- Rename attributes to conform to v2.
 \copy (SELECT user_id, username FROM users) TO users.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
-\copy (SELECT project_id, task_id, user_id, timestamp as "timeint", info FROM results WHERE project_id = 5549) TO results.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
+\copy (SELECT project_id, task_id, user_id, timestamp as "timeint", info FROM results WHERE project_id in (SELECT project_id FROM projects GROUP BY project_id) AND user_id in (SELECT user_id FROM users)) TO results.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);

postgres/scripts/v1_to_v2/copy_to_csv.sql

@@ -1,9 +1,6 @@
 -- Export v1 MapSwipe data to csv.
 -- Rename attributes to conform to v2.
-\copy (SELECT archive, image, importkey as "import_id", isfeatured AS "is_featured", lookfor AS "look_for", name, progress, projectdetails AS "project_details", project_id, project_type, state AS "status", info AS "project_type_specifics" FROM projects WHERE project_id in (select project_id from results group by project_id) AND project_id = 5549) TO projects.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
-
-\copy (SELECT i.import_id, i.info FROM imports i , projects p WHERE import_id in (select importkey as import_id from projects) AND p.project_id = 5549 AND p.importkey = i.import_id) TO imports.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
-
-\copy (SELECT project_id, group_id as "v1_group_id", count as "number_of_tasks", completedcount as "finished_count", verificationcount as "required_count", info as "project_type_specifics" FROM groups WHERE project_id in (select project_id from results group by project_id) AND project_id = 5549 ) TO groups.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
-
-\copy (SELECT project_id, group_id as "v1_group_id", task_id, info as "project_type_specifics" FROM tasks WHERE project_id in (select project_id from results group by project_id) AND project_id = 5549) TO tasks.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
+\copy (SELECT archive, image, importkey as "import_id", isfeatured AS "is_featured", lookfor AS "look_for", name, progress, projectdetails AS "project_details", project_id, project_type, state AS "status", info AS "project_type_specifics" FROM projects WHERE project_id in (select project_id from results group by project_id)) TO projects.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
+\copy (SELECT i.import_id, i.info FROM imports i WHERE import_id in (select importkey as import_id from projects)) TO imports.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
+\copy (SELECT project_id, group_id as "v1_group_id", count as "number_of_tasks", completedcount as "finished_count", verificationcount as "required_count", info as "project_type_specifics" FROM groups WHERE project_id in (select project_id from projects group by project_id) and project_id in (select project_id from results group by project_id) ) TO groups.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
+\copy (SELECT project_id, group_id as "v1_group_id", task_id, info as "project_type_specifics" FROM tasks WHERE project_id in (select project_id from projects group by project_id) and project_id in (select project_id from results group by project_id)) TO tasks.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);