Merge pull request #232 from mapswipe/feature-overall-stats

Feature overall stats

Author: Hagellach37

mapswipe_workers/mapswipe_workers/generate_stats/generate_stats.py

@@ -20,8 +20,8 @@ def generate_stats(project_id_list: list):
     ----------
     project_id_list: list
     """
-    logger.info(f"will generate stats for: {project_id_list}")
 
+    logger.info(f"will generate stats for: {project_id_list}")
     projects_info_filename = f"{DATA_PATH}/api-data/projects/projects_static.csv"
     projects_df = overall_stats.get_project_static_info(projects_info_filename)
     project_id_list_postgres = projects_df["project_id"].to_list()
@@ -60,10 +60,14 @@ def generate_stats(project_id_list: list):
 
             # TODO: for build area projects generate tasking manager geometries
 
-    # merge static info and dynamic info and save
     if len(project_id_list) > 0:
+        # merge static info and dynamic info and save
         projects_filename = f"{DATA_PATH}/api-data/projects/projects.csv"
-        overall_stats.save_projects(projects_filename, projects_df, projects_dynamic_df)
+        projects_df = overall_stats.save_projects(projects_filename, projects_df, projects_dynamic_df)
+
+        # generate overall stats for active, inactive, finished projects
+        overall_stats_filename = f"{DATA_PATH}/api-data/stats.csv"
+        overall_stats.get_overall_stats(projects_df, overall_stats_filename)
 
     logger.info(f"finished generate stats for: {project_id_list}")
 

mapswipe_workers/mapswipe_workers/generate_stats/overall_stats.py

@@ -5,18 +5,57 @@
 from mapswipe_workers.utils import geojson_functions
 
 
-def get_project_static_info(filename):
+def get_overall_stats(projects_df: pd.DataFrame, filename: str) -> pd.DataFrame:
+    """
+    The function aggregates the statistics per project using the status attribute.
+    We derive aggregated statistics for active, inactive and finished projects.
+    The number of users should not be summed up here, since this would generate wrong results.
+    A single user can contribute to multiple projects, we need to consider this.
+
+    Parameters
+    ----------
+    projects_df: pd.DataFrame
+    filename: str
+    """
+
+    overall_stats_df = projects_df.groupby(['status']).agg(
+        count_projects=pd.NamedAgg(column='project_id', aggfunc='count'),
+        area_sqkm=pd.NamedAgg(column='area_sqkm', aggfunc='sum'),
+        number_of_results=pd.NamedAgg(column='number_of_results', aggfunc='sum'),
+        number_of_results_progress=pd.NamedAgg(column='number_of_results_progress', aggfunc='sum'),
+        average_number_of_users_per_project=pd.NamedAgg(column='number_of_users', aggfunc='mean')
+    )
+
+    overall_stats_df.to_csv(filename, index_label="status")
+    logger.info(f'saved overall stats to {filename}')
+
+    return overall_stats_df
+
+
+def get_project_static_info(filename: str) -> pd.DataFrame:
+    """
+    The function queries the projects table.
+    Each row represents a single project and provides the information which is static.
+    By static we understand all attributes which are not affected by new results being contributed.
+    The results are stored in a csv file and also returned as a pandas DataFrame.
+
+    Parameters
+    ----------
+    filename: str
+    """
 
     pg_db = auth.postgresDB()
+
+    # make sure to replace newline characters here
     sql_query = """
         COPY (
             SELECT 
                 project_id
-                ,name
-                ,project_details
-                ,look_for
+                ,regexp_replace(name, E'[\\n\\r]+', ' ', 'g' ) as name
+                ,regexp_replace(project_details, E'[\\n\\r]+', ' ', 'g' ) as project_details
+                ,regexp_replace(look_for, E'[\\n\\r]+', ' ', 'g' ) as look_for
                 ,project_type
-                ,status
+                ,regexp_replace(status, E'[\\n\\r]+', ' ', 'g' ) as status
                 ,ST_Area(geom::geography)/1000000 as area_sqkm
                 ,ST_AsText(geom) as geom
                 ,ST_AsText(ST_Centroid(geom)) as centroid
@@ -30,10 +69,20 @@ def get_project_static_info(filename):
     logger.info("got projects from postgres.")
 
     df = pd.read_csv(filename)
+
     return df
 
 
-def load_project_info_dynamic(filename):
+def load_project_info_dynamic(filename: str) -> pd.DataFrame:
+    """
+    The function loads data from a csv file into a pandas dataframe.
+    If not file exists, it will be initialized.
+
+    Parameters
+    ----------
+    filename: str
+    """
+
     if os.path.isfile(filename):
         logger.info(f"file {filename} exists. Init from this file.")
         df = pd.read_csv(filename, index_col="idx")
@@ -52,11 +101,27 @@ def load_project_info_dynamic(filename):
     return df
 
 
-def save_projects(filename, df, df_dynamic):
+def save_projects(filename: str, df: pd.DataFrame, df_dynamic: pd.DataFrame) -> pd.DataFrame:
+    """
+    The function merges the dataframes for static and dynamic project information
+    and then save the result as csv file.
+    Additionally, two geojson files are generated using
+    (a) the geometry of the projects and
+    (b) the centroid of the projects.
+
+    Parameters
+    ----------
+    filename: str
+    df: pd.DataFrame
+    df_dynamic: pd.DataFrame
+    """
+
     projects_df = df.merge(
         df_dynamic, left_on="project_id", right_on="project_id", how="left"
     )
-    projects_df.to_csv(filename, index_label="idx")
+    projects_df.to_csv(filename, index_label="idx", line_terminator='\n')
     logger.info(f"saved projects: {filename}")
     geojson_functions.csv_to_geojson(filename, "geom")
     geojson_functions.csv_to_geojson(filename, "centroid")
+
+    return projects_df