Merge branch 'dev' of github.com:mapswipe/python-mapswipe-workers into dev

Author: Hagellach37

README.md

@@ -10,7 +10,7 @@ The MapSwipe Back-End consists of a number of components:
 3. Manager Dashboard
 5. API
 
-Please refer to the documentation for more information: https://mapswipe-workers.readthedocs.io/en/dev/
+Please refer to the documentation for more information: https://mapswipe-workers.readthedocs.io/en/
 
 
 ## Ressources
@@ -26,7 +26,6 @@ Please refer to the documentation for more information: https://mapswipe-workers
 
 This project is licensed under the Apache License 2.0 - see the [LICENSE](LICENSE) file for details
 
-
 ## Authors
 
 * **Benjamin Herfort** - HeiGIT - [Hagellach37](https://github.com/Hagellach37)
@@ -35,7 +34,6 @@ This project is licensed under the Apache License 2.0 - see the [LICENSE](LICENS
 
 See also the list of [contributors](contributors.md) who participated in this project.
 
-
 ## Acknowledgements
 
-> Humanitarian organizations can't help people if they can't find them.
+* Humanitarian organizations can't help people if they can't find them.

postgres/scripts/v1_to_v2/copy_from_csv.sh

@@ -2,11 +2,13 @@
 
 USER="mapswipe_workers"
 NAME="mapswipe"
-#tasks users results
-for entity in users results
+
+for entity in projects groups tasks users results
 do
     psql -h localhost -p 5432 -U ${USER} -d ${NAME} -a -f copy_${entity}_from_csv.sql
 done
+
+# Commands if docker postgres port is not exposed:
 # docker cp ${entity}.csv mapswipe_postgres:${entity}.csv
 # docker cp copy_${entity}_from_csv.sql mapswipe_postgres:copy_${entity}_from_csv.sql
 # docker exec -t mapswipe_postgres psql -U ${USER} -d ${NAME} -a -f copy_${entity}_from_csv.sql

postgres/scripts/v1_to_v2/copy_groups_from_csv.sql

@@ -25,6 +25,16 @@ UPDATE v1_groups
 SET progress = 100
 WHERE required_count <= 0;
 
+UPDATE v1_groups
+SET group_id = cast(v1_group_id as varchar);
+
+UPDATE groups
+SET (finished_count, required_count, progress) =
+        (SELECT finished_count, required_count, progress
+        FROM v1_groups
+        WHERE groups.group_id = v1_groups.group_id
+        AND groups.project_id = v1_groups.project_id);
+
 -- Insert or update data of temp table to the permant table.
 -- Note that the special excluded table is used to
 -- reference values originally proposed for insertion
@@ -48,9 +58,4 @@ SELECT
   project_type_specifics
 FROM
   v1_groups
-ON CONFLICT (project_id, group_id) DO UPDATE
-SET
-  finished_count = excluded.finished_count,
-  required_count = excluded.required_count,
-  progress = excluded.progress
-;
+ON CONFLICT (project_id, group_id) DO NOTHING;

postgres/scripts/v1_to_v2/copy_projects_from_csv.sql

@@ -45,12 +45,9 @@ SET status = 'inactive'
 WHERE status = '3';
 
 -- Extract geometry (kml) from info attribute (json) of imports table.
-UPDATE v1_imports i1
-SET kml = (SELECT info->'info'->>'kml' as kml FROM v1_imports i2 WHERE i1.import_id = i2.import_id);
-
 -- Get only the geometry (Polygone or MultiPolygon) of the kml string.
 UPDATE v1_imports
-SET kml = substring(kml FROM '<Polygon>.*<\/Polygon>|<MultiPolygon>.*<\/MultiPolygon>');
+SET kml = substring(info->'info'->>'kml' FROM '<Polygon>.*<\/Polygon>|<MultiPolygon>.*<\/MultiPolygon>');
 
 UPDATE v1_projects
 SET kml = v1_imports.kml

postgres/scripts/v1_to_v2/copy_results_to_csv.sql

@@ -1,4 +1,4 @@
 -- Export v1 MapSwipe data to csv.
 -- Rename attributes to conform to v2.
 \copy (SELECT user_id, username FROM users) TO users.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
-\copy (SELECT project_id, task_id, user_id, timestamp as "timeint", info FROM results WHERE project_id = 5519) TO results.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
+\copy (SELECT project_id, task_id, user_id, timestamp as "timeint", info FROM results WHERE project_id = 5549) TO results.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);

postgres/scripts/v1_to_v2/copy_to_csv.sql

@@ -1,6 +1,6 @@
 -- Export v1 MapSwipe data to csv.
 -- Rename attributes to conform to v2.
-\copy (SELECT archive, image, importkey as "import_id", isfeatured AS "is_featured", lookfor AS "look_for", name, progress, projectdetails AS "project_details", project_id, project_type, state AS "status", info AS "project_type_specifics" FROM projects WHERE project_id = 5519) TO projects.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
-\copy (SELECT i.import_id, i.info FROM imports i, projects p WHERE p.project_id = 5519 AND p.importkey = i.import_id) TO imports.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
-\copy (SELECT project_id, group_id, count as "number_of_tasks", completedcount as "finished_count", verificationcount as "required_count", info as "project_type_specifics" FROM groups WHERE project_id = 5519 ) TO groups.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
-\copy (SELECT project_id, group_id, task_id, info as "project_type_specifics" FROM tasks WHERE project_id = 5519) TO tasks.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
+\copy (SELECT archive, image, importkey as "import_id", isfeatured AS "is_featured", lookfor AS "look_for", name, progress, projectdetails AS "project_details", project_id, project_type, state AS "status", info AS "project_type_specifics" FROM projects WHERE project_id in (select project_id from results group by project_id) AND project_id = 5549) TO projects.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
+\copy (SELECT i.import_id, i.info FROM imports i , projects p WHERE import_id in (select importkey as import_id from projects) AND p.project_id = 5549 AND p.importkey = i.import_id) TO imports.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
+\copy (SELECT project_id, group_id, count as "number_of_tasks", completedcount as "finished_count", verificationcount as "required_count", info as "project_type_specifics" FROM groups WHERE project_id in (select project_id from results group by project_id) AND project_id = 5549 ) TO groups.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);
+\copy (SELECT project_id, group_id, task_id, info as "project_type_specifics" FROM tasks WHERE project_id in (select project_id from results group by project_id) AND project_id = 5549) TO tasks.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);

postgres/scripts/v1_to_v2/generate_copy_to_csv.py

@@ -2,35 +2,37 @@
 
 
 def get_query(project_ids):
+    clause = 'WHERE project_id in (select project_id from results group by project_id)'
+    clause_import = 'WHERE import_id in (select importkey as import_id from projects)'
     if project_ids is None:
-        clause = ''
-        clause_import = ''
+        pass
     elif len(project_ids) == 1:
-        clause = f'WHERE project_id = {project_ids[0]}'
-        clause_import = f'WHERE p.project_id = {project_ids[0]} AND p.importkey = i.import_id'
+        clause = f'{clause} AND project_id = {project_ids[0]}'
+        clause_import = f', projects p {clause_import} AND p.project_id = {project_ids[0]} AND p.importkey = i.import_id'
     else:
-        clause = f'WHERE project_id = {project_ids[0]}'
-        clause_import = f'WHERE p.project_id = {project_ids[0]} AND p.importkey = i.import_id'
+        clause = f'{clause} AND project_id = {project_ids[0]}'
+        clause_import = f', projects p {clause_import} AND p.project_id = {project_ids[0]} AND p.importkey = i.import_id'
         for project_id in project_ids[1:]:
             clause = clause + f' OR project_id = {project_id}'
             clause_import = clause + f' OR p.project_id = {project_id} AND p.importkey = i.import_id'
 
     query = f'-- Export v1 MapSwipe data to csv.\n' \
             f'-- Rename attributes to conform to v2.\n' \
             f'\copy (SELECT archive, image, importkey as "import_id", isfeatured AS "is_featured", lookfor AS "look_for", name, progress, projectdetails AS "project_details", project_id, project_type, state AS "status", info AS "project_type_specifics" FROM projects {clause}) TO projects.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);\n' \
-            f'\copy (SELECT i.import_id, i.info FROM imports i, projects p {clause_import}) TO imports.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);\n' \
-            f'\copy (SELECT project_id, group_id, count as "number_of_tasks", completedcount as "finished_count", verificationcount as "required_count", info as "project_type_specifics" FROM groups {clause} ) TO groups.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);\n' \
+            f'\copy (SELECT i.import_id, i.info FROM imports i {clause_import}) TO imports.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);\n' \
+            f'\copy (SELECT project_id, group_id as "v1_group_id", count as "number_of_tasks", completedcount as "finished_count", verificationcount as "required_count", info as "project_type_specifics" FROM groups {clause} ) TO groups.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);\n' \
             f'\copy (SELECT project_id, group_id, task_id, info as "project_type_specifics" FROM tasks {clause}) TO tasks.csv WITH (FORMAT CSV, DELIMITER ",", HEADER TRUE);\n'
     return query
 
 
 def get_result_query(project_ids):
+    clause = 'WHERE project_id in (SELECT project_id FROM projects GROUP BY project_id)'
     if project_ids is None:
-         clause = ''
+        pass
     elif len(project_ids) == 1:
-        clause = f'WHERE project_id = {project_ids[0]}'
+        clause = f'{clause} AND project_id = {project_ids[0]}'
     else:
-        clause = f'WHERE project_id = {project_ids[0]}'
+        clause = f'{clause} AND project_id = {project_ids[0]}'
         for project_id in project_ids[1:]:
             clause = clause + f' OR project_id = {project_id}'