add option to generate stats for all projects

Author: Hagellach37

mapswipe_workers/mapswipe_workers/generate_stats/generate_stats.py

@@ -5,15 +5,20 @@
 from mapswipe_workers.generate_stats import overall_stats
 
 
-def generate_stats(project_id_list):
-    logger.info(f'will generate stats for: {project_id_list}')
+def generate_stats(project_id_list, all_projects=False):
 
     projects_info_filename = f'{DATA_PATH}/api-data/projects/projects_static.csv'
     projects_df = overall_stats.get_project_static_info(projects_info_filename)
 
     projects_info_dynamic_filename = f'{DATA_PATH}/api-data/projects/projects_dynamic.csv'
     projects_dynamic_df = overall_stats.load_project_info_dynamic(projects_info_dynamic_filename)
 
+    if all_projects:
+        project_id_list = projects_df['project_id'].to_list()
+        logger.info(f'will generate stats for all projects.')
+
+    logger.info(f'will generate stats for: {project_id_list}')
+
     # get per project stats and aggregate based on task_id
     for project_id in project_id_list:
 
@@ -29,10 +34,11 @@ def generate_stats(project_id_list):
 
         # aggregate results and get per project statistics
         project_stats_dict = project_stats.get_per_project_statistics(project_id)
-        projects_dynamic_df = projects_dynamic_df.append(project_stats_dict, ignore_index=True)
-        projects_dynamic_df.to_csv(projects_info_dynamic_filename, index_label='idx')
+        if project_stats_dict:
+            projects_dynamic_df = projects_dynamic_df.append(project_stats_dict, ignore_index=True)
+            projects_dynamic_df.to_csv(projects_info_dynamic_filename, index_label='idx')
 
-        # TODO: for build area projects generate tasking manager geometries
+            # TODO: for build area projects generate tasking manager geometries
 
     # merge static info and dynamic info and save
     if len(project_id_list) > 0:

mapswipe_workers/mapswipe_workers/generate_stats/project_stats.py

@@ -38,14 +38,18 @@ def get_results_by_project_id(filename, project_id):
     logger.info(f'got results from postgres for {project_id}')
 
     df = pd.read_csv(filename)
-    df['group_id'] = df.apply(lambda row: id_to_string(row['group_id']), axis=1)
-    df['group_id'] = df['group_id'].astype(str)
-    df['timestamp'] = pd.to_datetime(df['timestamp'])
-    df['day'] = df['timestamp'].apply(
-        lambda df: datetime.datetime(year=df.year, month=df.month, day=df.day))
 
-    logger.info(f'created pandas results df for {project_id}')
-    return df
+    if len(df) > 0:
+        df['group_id'] = df.apply(lambda row: id_to_string(row['group_id']), axis=1)
+        df['group_id'] = df['group_id'].astype(str)
+        df['timestamp'] = pd.to_datetime(df['timestamp'])
+        df['day'] = df['timestamp'].apply(
+            lambda df: datetime.datetime(year=df.year, month=df.month, day=df.day))
+        logger.info(f'created pandas results df for {project_id}')
+        return df
+    else:
+        logger.info(f'there are no results for this project {project_id}')
+        return None
 
 
 def get_tasks_by_project_id(filename, project_id):
@@ -246,34 +250,40 @@ def get_per_project_statistics(project_id):
 
     # load data from postgres or local storage if already downloaded
     results_df = get_results_by_project_id(results_filename, project_id)
-    groups_df = get_groups_by_project_id(groups_filename, project_id)
-    tasks_df = get_tasks_by_project_id(tasks_filename, project_id)
-
-    # aggregate results by task id
-    agg_results_df = agg_results_by_task_id(results_df, tasks_df)
-    agg_results_df.to_csv(agg_results_filename, index_label='idx')
-    logger.info(f'saved agg results for {project_id}: {agg_results_filename}')
-    geojson_functions.csv_to_geojson(agg_results_filename, 'geom')
-
-    # calculate progress by date
-    progress_by_date_df = get_progress_by_date(results_df, groups_df)
-
-    # calculate contributors by date
-    contributors_by_date_df = get_contributors_by_date(results_df)
-
-    # merge contributors and progress
-    project_stats_by_date_df = progress_by_date_df.merge(contributors_by_date_df, left_on='day', right_on='day')
-    project_stats_by_date_df['project_id'] = project_id
-    project_stats_by_date_df.to_csv(project_stats_by_date_filename)
-    logger.info(f'saved project stats by date for {project_id}: {project_stats_by_date_filename}')
-
-    project_stats_dict = {
-        'project_id': project_id,
-        'progress': project_stats_by_date_df['cum_progress'].iloc[-1],
-        'number_of_users': project_stats_by_date_df['cum_number_of_users'].iloc[-1],
-        'number_of_results': project_stats_by_date_df['cum_number_of_results'].iloc[-1],
-        'number_of_results_progress': project_stats_by_date_df['cum_number_of_results_progress'].iloc[-1],
-        'day': project_stats_by_date_df.index[-1]
-    }
-
-    return project_stats_dict
+
+    if results_df is None:
+        logger.info(f'no results: skipping per project stats for {project_id}')
+        return None
+    else:
+        groups_df = get_groups_by_project_id(groups_filename, project_id)
+        tasks_df = get_tasks_by_project_id(tasks_filename, project_id)
+
+        # aggregate results by task id
+        agg_results_df = agg_results_by_task_id(results_df, tasks_df)
+        agg_results_df.to_csv(agg_results_filename, index_label='idx')
+        logger.info(f'saved agg results for {project_id}: {agg_results_filename}')
+        geojson_functions.csv_to_geojson(agg_results_filename, 'geom')
+
+        # calculate progress by date
+        progress_by_date_df = get_progress_by_date(results_df, groups_df)
+
+        # calculate contributors by date
+        contributors_by_date_df = get_contributors_by_date(results_df)
+
+        # merge contributors and progress
+        project_stats_by_date_df = progress_by_date_df.merge(contributors_by_date_df, left_on='day', right_on='day')
+        project_stats_by_date_df['project_id'] = project_id
+        project_stats_by_date_df.to_csv(project_stats_by_date_filename)
+        logger.info(f'saved project stats by date for {project_id}: {project_stats_by_date_filename}')
+
+        project_stats_dict = {
+            'project_id': project_id,
+            'progress': project_stats_by_date_df['cum_progress'].iloc[-1],
+            'number_of_users': project_stats_by_date_df['cum_number_of_users'].iloc[-1],
+            'number_of_results': project_stats_by_date_df['cum_number_of_results'].iloc[-1],
+            'number_of_results_progress': project_stats_by_date_df['cum_number_of_results_progress'].iloc[-1],
+            'day': project_stats_by_date_df.index[-1]
+        }
+
+        return project_stats_dict
+

mapswipe_workers/mapswipe_workers/mapswipe_workers.py

@@ -156,37 +156,48 @@ def run_firebase_to_postgres(schedule):
 @click.option(
         '--project_id_list',
         cls=PythonLiteralOption,
-        default=[],
+        default='[]',
         help=(
             f'provide project id strings as a list '
             f'stats will be generated only for this'
             f'''use it like '["project_a", "project_b"]' '''
             )
         )
-def run_generate_stats(schedule, project_id_list):
+@click.option(
+        '--all_projects',
+        default=False,
+        is_flag=True,
+        help=(
+            f'Generate stats for all projects in postgres'
+            )
+        )
+def run_generate_stats(schedule, project_id_list, all_projects):
     sentry.init_sentry()
     try:
         if schedule:
             if schedule == 'm':
                 sched.every(10).minutes.do(
                         _run_generate_stats,
-                        project_id_list=project_id_list
+                        project_id_list=project_id_list,
+                        all_projects=all_projects
                         ).run()
                 while True:
                     sched.run_pending()
                     time.sleep(1)
             elif schedule == 'h':
                 sched.every().hour.do(
                         _run_generate_stats,
-                        project_id_list=project_id_list
+                        project_id_list=project_id_list,
+                        all_projects=all_projects
                         ).run()
                 while True:
                     sched.run_pending()
                     time.sleep(1)
             elif schedule == 'd':
                 sched.every().day.do(
                         _run_generate_stats,
-                        project_id_list=project_id_list
+                        project_id_list=project_id_list,
+                        all_projects=all_projects
                         ).run()
                 while True:
                     sched.run_pending()
@@ -199,7 +210,7 @@ def run_generate_stats(schedule, project_id_list):
                         f'h for every hour and d for every day.'
                         )
         else:
-            _run_generate_stats(project_id_list)
+            _run_generate_stats(project_id_list, all_projects)
     except Exception as e:
         slack.send_error(e)
         sentry.capture_exception_sentry(e)
@@ -421,8 +432,8 @@ def _run_firebase_to_postgres():
     return project_id_list
 
 
-def _run_generate_stats(project_id_list):
-    generate_stats.generate_stats(project_id_list)
+def _run_generate_stats(project_id_list, all_projects):
+    generate_stats.generate_stats(project_id_list, all_projects)
 
 
 def _run_user_management(email, manager):