use temporary file for sql query results

Author: Hagellach37

mapswipe_workers/mapswipe_workers/generate_stats/project_stats.py

@@ -2,7 +2,7 @@
 import os
 from typing import List
 import gzip
-import io
+import tempfile
 
 import pandas as pd
 from psycopg2 import sql
@@ -28,22 +28,20 @@ def add_metadata_to_csv(filename: str):
     logger.info(f"added metadata to {filename}.")
 
 
-def write_sql_to_csv(filename: str, sql_query: sql.SQL):
+def write_sql_to_gzipped_csv(filename: str, sql_query: sql.SQL):
     """
     Use the copy statement to write data from postgres to a csv file.
     """
 
-    temp_file = "temp.csv"
+    # generate temporary file which will be automatically deleted at the end
+    tmp_csv_file = os.path.join(tempfile._get_default_tempdir(), 'tmp.csv')
     pg_db = auth.postgresDB()
-    with open(temp_file, "w") as f:
+    with open(tmp_csv_file, "w") as f:
         pg_db.copy_expert(sql_query, f)
 
-    with open(temp_file, 'rb') as f_in, gzip.open(filename, 'wb') as f_out:
+    with open(tmp_csv_file, 'rb') as f_in, gzip.open(filename, 'wb') as f_out:
         f_out.writelines(f_in)
 
-    # remove temp file
-    os.remove(temp_file)
-
     logger.info(f"wrote gzipped csv file from sql: {filename}")
 
 
@@ -87,7 +85,7 @@ def get_results(filename: str, project_id: str) -> pd.DataFrame:
         ) TO STDOUT WITH CSV HEADER
         """
     ).format(sql.Literal(project_id))
-    write_sql_to_csv(filename, sql_query)
+    write_sql_to_gzipped_csv(filename, sql_query)
 
     df = load_df_from_csv(filename)
 
@@ -131,7 +129,7 @@ def get_tasks(filename: str, project_id: str) -> pd.DataFrame:
             ) TO STDOUT WITH CSV HEADER
             """
         ).format(sql.Literal(project_id))
-        write_sql_to_csv(filename, sql_query)
+        write_sql_to_gzipped_csv(filename, sql_query)
 
     df = load_df_from_csv(filename)
     return df
@@ -166,7 +164,7 @@ def get_groups(filename: str, project_id: str) -> pd.DataFrame:
             ) TO STDOUT WITH CSV HEADER
             """
         ).format(sql.Literal(project_id))
-        write_sql_to_csv(filename, sql_query)
+        write_sql_to_gzipped_csv(filename, sql_query)
 
     df = load_df_from_csv(filename)
     return df

mapswipe_workers/mapswipe_workers/utils/geojson_functions.py

@@ -4,7 +4,6 @@
 import shutil
 import subprocess
 import tempfile
-from pathlib import Path
 
 from osgeo import ogr, osr
 
@@ -22,7 +21,7 @@ def gzipped_csv_to_gzipped_geojson(
     Then the unzipped csv file is converted into a geojson file with ogr2ogr.
     Last, the generated geojson file is again compressed using gzip.
     """
-    # generate tempory files which will be automatically deleted at the end
+    # generate temporary files which will be automatically deleted at the end
     tmp_csv_file = os.path.join(tempfile._get_default_tempdir(), 'tmp.csv')
     tmp_geojson_file = os.path.join(tempfile._get_default_tempdir(), 'tmp.geojson')