work on PR comments #218

Author: Hagellach37

mapswipe_workers/mapswipe_workers/firebase_to_postgres/update_data.py

@@ -5,27 +5,27 @@
 
 
 def update_user_data(user_ids=None):
-    '''
+    """
     Copies new users from Firebase to Postgres
-    '''
+    """
     # TODO: On Conflict
     fb_db = auth.firebaseDB()
     pg_db = auth.postgresDB()
 
-    fb_ref = fb_db.reference('v2/users')
+    fb_ref = fb_db.reference("v2/users")
 
-    pg_query = '''
+    pg_query = """
         SELECT created
         FROM users
         ORDER BY created DESC
         LIMIT 1
-        '''
+        """
     last_updated = pg_db.retr_query(pg_query)
     try:
         last_updated = last_updated[0][0]
-        logger.info(f'got last updated timestamp: {last_updated}')
+        logger.info(f"got last updated timestamp: {last_updated}")
     except:
-        logger.info('could not get last timestamp')
+        logger.info("could not get last timestamp")
         last_updated = None
 
     if last_updated is None:
@@ -34,13 +34,15 @@ def update_user_data(user_ids=None):
         users = fb_ref.get()
     else:
         # Get only new users from Firebase.
-        last_updated = last_updated.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
-        fb_query = fb_ref.order_by_child('created').start_at(last_updated)
+        last_updated = last_updated.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+        fb_query = fb_ref.order_by_child("created").start_at(last_updated)
         users = fb_query.get()
         # Delete first user in ordered dict.
         # This user is already in the database (user.created = last_updated).
         if len(users) == 0:
-            logger.info(f"there are no new users in firebase based on created timestamp")
+            logger.info(
+                f"there are no new users in firebase based on created timestamp"
+            )
         else:
             users.popitem(last=False)
 
@@ -57,40 +59,43 @@ def update_user_data(user_ids=None):
             # Convert timestamp (ISO 8601) from string to a datetime object.
             try:
                 created = dt.datetime.strptime(
-                        user['created'].replace('Z', ''),
-                        '%Y-%m-%dT%H:%M:%S.%f'
-                        )
+                    user["created"].replace("Z", ""), "%Y-%m-%dT%H:%M:%S.%f"
+                )
             except KeyError:
                 # if user has no "created" attribute, we set it to current time
-                created = dt.datetime.utcnow().isoformat()[0:-3]+'Z'
-                logger.info(f"user {user_id} didn't have a created attribute set it to {created}")
+                created = dt.datetime.utcnow().isoformat()[0:-3] + "Z"
+                logger.info(
+                    f"user {user_id} didn't have a 'created' attribute. Set it to '{created}' now."
+                )
 
             try:
-                username = user['username']
+                username = user["username"]
             except KeyError:
                 # if user has no "username" attribute, we set it to None
                 username = None
-                logger.info(f"user {user_id} didn't have a username attribute set it to {username}")
+                logger.info(
+                    f"user {user_id} didn't have a 'username' attribute. Set it to '{username}' now."
+                )
 
-            query_update_user = '''
+            query_update_user = """
                 INSERT INTO users (user_id, username, created)
                 VALUES(%s, %s, %s)
                 ON CONFLICT (user_id) DO UPDATE
                 SET username=%s,
                 created=%s;
-            '''
+            """
             data_update_user = [
-                    user_id,
-                    username,
-                    created,
-                    username,
-                    created,
-                    ]
+                user_id,
+                username,
+                created,
+                username,
+                created,
+            ]
             pg_db.query(query_update_user, data_update_user)
 
-    del(pg_db)
+    del pg_db
 
-    logger.info('Updated user data in Potgres')
+    logger.info("Updated user data in Potgres")
 
 
 def update_project_data(project_ids=None):
@@ -113,26 +118,26 @@ def update_project_data(project_ids=None):
         logger.info(f"update project data in postgres for selected projects")
         projects = dict()
         for project_id in project_ids:
-            project_ref = fb_db.reference(f'v2/projects/{project_id}')
+            project_ref = fb_db.reference(f"v2/projects/{project_id}")
             projects[project_id] = project_ref.get()
     else:
         logger.info(f"update project data in postgres for all firebase projects")
-        projects_ref = fb_db.reference('v2/projects/')
+        projects_ref = fb_db.reference("v2/projects/")
         projects = projects_ref.get()
 
     if projects:
         for project_id, project in projects.items():
-            query_update_project = '''
+            query_update_project = """
                 UPDATE projects
                 SET status=%s
                 WHERE project_id=%s;
-            '''
+            """
             # TODO: Is there need for fallback to ''
             # if project.status is not existent
-            data_update_project = [project.get('status', ''), project_id]
+            data_update_project = [project.get("status", ""), project_id]
             pg_db.query(query_update_project, data_update_project)
             logger.info(f"updated status for project {project_id} in postgres")
 
-    del(pg_db)
+    del pg_db
 
-    logger.info('Updated project data in Postgres')
+    logger.info("Updated project data in Postgres")

mapswipe_workers/mapswipe_workers/generate_stats/generate_stats.py

@@ -1,46 +1,84 @@
 from mapswipe_workers.definitions import logger
 from mapswipe_workers.definitions import DATA_PATH
+from mapswipe_workers.generate_stats import project_stats, overall_stats
 
-from mapswipe_workers.generate_stats import project_stats
-from mapswipe_workers.generate_stats import overall_stats
 
+def generate_stats(project_id_list: list):
+    """
+    Query attributes for all projects from postgres projects table
+    Write information on status (e.g. active, inactive, finished) and further attributes
+    for all projects to projects_static.csv.
+    Computationally more expensive tasks are only performed for projects specified in project_id_list.
+    Write information on progress and contributors and further attributes
+    only for projects specified in project_id_list to projects_dynamic.csv.
+    Write information on project progress history and aggregated results
+    only for projects specified in project_id_list to csv and geojson files.
+    Merge projects_static.csv and projects_dynamic.csv into projects.csv.
+    Convert projects.csv file into GeoJSON format using project geometry and project centroid.
 
-def generate_stats(project_id_list, all_projects=False):
+    Parameters
+    ----------
+    project_id_list: list
+    """
+    logger.info(f"will generate stats for: {project_id_list}")
 
-    projects_info_filename = f'{DATA_PATH}/api-data/projects/projects_static.csv'
+    projects_info_filename = f"{DATA_PATH}/api-data/projects/projects_static.csv"
     projects_df = overall_stats.get_project_static_info(projects_info_filename)
+    project_id_list_postgres = projects_df["project_id"].to_list()
 
-    projects_info_dynamic_filename = f'{DATA_PATH}/api-data/projects/projects_dynamic.csv'
-    projects_dynamic_df = overall_stats.load_project_info_dynamic(projects_info_dynamic_filename)
-
-    if all_projects:
-        project_id_list = projects_df['project_id'].to_list()
-        logger.info(f'will generate stats for all projects.')
-
-    logger.info(f'will generate stats for: {project_id_list}')
+    projects_info_dynamic_filename = (
+        f"{DATA_PATH}/api-data/projects/projects_dynamic.csv"
+    )
+    projects_dynamic_df = overall_stats.load_project_info_dynamic(
+        projects_info_dynamic_filename
+    )
 
     # get per project stats and aggregate based on task_id
     for project_id in project_id_list:
 
-        # check if project id is correct
-        if not project_id in projects_df['project_id'].to_list():
-            logger.info(f'project {project_id} does not exist. skip this one.')
+        # check if project id is existing
+        if project_id not in project_id_list_postgres:
+            logger.info(f"project {project_id} does not exist. skip this one.")
             continue
 
-        logger.info(f'start generate stats for project: {project_id}')
-        idx = projects_dynamic_df.index[projects_dynamic_df['project_id'] == project_id].tolist()
+        logger.info(f"start generate stats for project: {project_id}")
+        idx = projects_dynamic_df.index[
+            projects_dynamic_df["project_id"] == project_id
+        ].tolist()
         if len(idx) > 0:
             projects_dynamic_df.drop([idx[0]], inplace=True)
 
         # aggregate results and get per project statistics
         project_stats_dict = project_stats.get_per_project_statistics(project_id)
         if project_stats_dict:
-            projects_dynamic_df = projects_dynamic_df.append(project_stats_dict, ignore_index=True)
-            projects_dynamic_df.to_csv(projects_info_dynamic_filename, index_label='idx')
+            projects_dynamic_df = projects_dynamic_df.append(
+                project_stats_dict, ignore_index=True
+            )
+            projects_dynamic_df.to_csv(
+                projects_info_dynamic_filename, index_label="idx"
+            )
 
             # TODO: for build area projects generate tasking manager geometries
 
     # merge static info and dynamic info and save
     if len(project_id_list) > 0:
-        projects_filename = f'{DATA_PATH}/api-data/projects/projects.csv'
-        overall_stats.save_projects(projects_filename, projects_df, projects_dynamic_df)
+        projects_filename = f"{DATA_PATH}/api-data/projects/projects.csv"
+        overall_stats.save_projects(projects_filename, projects_df, projects_dynamic_df)
+
+
+def generate_stats_all_projects():
+    """
+    queries all existing project ids from postgres projects table
+    saves them into a csv file and returns a list of all project ids
+    then generates project statistics using the derived list of project ids
+    """
+
+    logger.info(f"will generate stats for all projects.")
+
+    # get all project ids from postgres database
+    projects_info_filename = f"{DATA_PATH}/api-data/projects/projects_static.csv"
+    projects_df = overall_stats.get_project_static_info(projects_info_filename)
+    project_id_list = projects_df["project_id"].to_list()
+
+    # generate stats for the derived project ids
+    generate_stats(project_id_list)

mapswipe_workers/mapswipe_workers/generate_stats/overall_stats.py

@@ -23,31 +23,40 @@ def get_project_static_info(filename):
             FROM projects
         ) TO STDOUT WITH CSV HEADER"""
 
-    with open(filename, 'w') as f:
+    with open(filename, "w") as f:
         pg_db.copy_expert(sql_query, f)
 
     del pg_db
-    logger.info('got projects from postgres.')
+    logger.info("got projects from postgres.")
 
     df = pd.read_csv(filename)
     return df
 
 
 def load_project_info_dynamic(filename):
     if os.path.isfile(filename):
-        logger.info(f'file {filename} exists. Init from this file.')
-        df = pd.read_csv(filename, index_col='idx')
+        logger.info(f"file {filename} exists. Init from this file.")
+        df = pd.read_csv(filename, index_col="idx")
     else:
-        columns = ['project_id', 'progress', 'number_of_users', 'number_of_results', 'number_of_results_progress', 'day']
+        columns = [
+            "project_id",
+            "progress",
+            "number_of_users",
+            "number_of_results",
+            "number_of_results_progress",
+            "day",
+        ]
         df = pd.DataFrame(index=[], columns=columns)
-        df['project_id'].astype('str')
+        df["project_id"].astype("str")
 
     return df
 
 
 def save_projects(filename, df, df_dynamic):
-    projects_df = df.merge(df_dynamic, left_on='project_id', right_on='project_id', how='left')
-    projects_df.to_csv(filename, index_label='idx')
-    logger.info(f'saved projects: {filename}')
-    geojson_functions.csv_to_geojson(filename, 'geom')
-    geojson_functions.csv_to_geojson(filename, 'centroid')
+    projects_df = df.merge(
+        df_dynamic, left_on="project_id", right_on="project_id", how="left"
+    )
+    projects_df.to_csv(filename, index_label="idx")
+    logger.info(f"saved projects: {filename}")
+    geojson_functions.csv_to_geojson(filename, "geom")
+    geojson_functions.csv_to_geojson(filename, "centroid")